framework,version,device,op_name,kernel_source,gemm_dtype,m,n,k,latency
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,4096,4.875103950500488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,5120,6.0003838539123535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,7168,8.27353572845459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,3584,4.304128170013428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,8192,11.413984298706055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,3072,3.7376320362091064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,1024,1.4758720397949219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,2560,3.158655881881714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,1536,2.0743680000305176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,2048,2.5828158855438232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,10240,20.866464614868164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,256,0.5988799929618835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,512,0.8883519768714905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,128,0.450111985206604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,12288,14.25926399230957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,8192,2.3728320598602295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,10240,2.938271999359131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,12288,3.500960111618042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,5120,1.523743987083435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,4096,1.246559977531433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,7168,2.093280076980591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,3584,1.0977280139923096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,16384,4.627488136291504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,3072,0.955839991569519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,2560,0.8234559893608093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,2048,0.6690559983253479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,512,0.23686400055885315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,1536,0.5202239751815796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,1024,0.3776639997959137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,256,0.15279999375343323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,128,0.11475200206041336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,16384,20.29248046875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,8192,1.7867200374603271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,7168,1.5732159614562988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,10240,2.208319902420044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,12288,2.6349759101867676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,4096,0.9373440146446228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,5120,1.1499199867248535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,3584,0.8324159979820251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,3072,0.7249280214309692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,2560,0.6153280138969421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,2048,0.5072320103645325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,1536,0.3985599875450134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,128,0.08774399757385254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,16384,3.478912115097046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,512,0.18406400084495544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,256,0.11580800265073776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,1024,0.2895039916038513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,8192,1.4915200471878052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,10240,1.8466559648513794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,12288,2.203968048095703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,7168,1.3146560192108154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,4096,0.7832000255584717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,5120,0.96288001537323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,16384,2.9148480892181396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,2560,0.5156480073928833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,3584,0.6967039704322815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,2048,0.4257279932498932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,3072,0.6077759861946106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,256,0.09849599748849869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,512,0.15545600652694702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,1024,0.24876800179481506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,1536,0.3353919982910156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,128,0.07401599735021591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,10240,1.5144319534301758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,12288,1.8068480491638184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,16384,2.388576030731201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,8192,1.2257599830627441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,7168,1.0819200277328491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,5120,0.7863680124282837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,4096,0.6412799954414368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,3584,0.5713279843330383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,3072,0.4989120066165924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,2048,0.3491840064525604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,2560,0.4239040017127991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,1536,0.2712000012397766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,512,0.12559999525547028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,1024,0.19884799420833588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,256,0.08025600016117096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,128,0.06019200012087822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,12288,1.5816960334777832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,65536,21.613248825073242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,16384,2.0937600135803223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,8192,1.0738879442214966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,65536,16.223072052001953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,10240,1.3267840147018433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,7168,0.9472960233688354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,4096,0.5628160238265991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,3584,0.5023999810218811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,5120,0.6912000179290771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,3072,0.43987199664115906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,1536,0.2396479994058609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,2560,0.37327998876571655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,2048,0.30851200222969055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,1024,0.17587199807167053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,512,0.11235199868679047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,128,0.05523199960589409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,256,0.07280000299215317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,65536,13.76364803314209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,10240,0.9588159918785095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,8192,0.7813760042190552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,12288,1.1422719955444336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,16384,1.497439980506897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,4096,0.4089280068874359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,5120,0.504863977432251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,7168,0.68476802110672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,65536,10.635199546813965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,3072,0.31942400336265564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,2560,0.2711679935455322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,3584,0.36287999153137207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,1536,0.1746239960193634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,2048,0.22409600019454956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,256,0.056543998420238495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,512,0.08371199667453766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,128,0.040991999208927155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,1024,0.1284160017967224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,8192,0.6254720091819763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,10240,0.7639359831809998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,12288,0.9101120233535767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,16384,1.2010879516601562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,5120,0.401311993598938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,7168,0.5494080185890198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,4096,0.3272640109062195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,2048,0.18105599284172058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,2560,0.2163199931383133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,3072,0.2548159956932068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,3584,0.2922559976577759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,512,0.06809599697589874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,256,0.04479999840259552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,1024,0.10480000078678131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,1536,0.14105600118637085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,128,0.03561599925160408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,65536,9.701600074768066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,10240,0.6714239716529846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,12288,0.792639970779419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,8192,0.5390400290489197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,16384,1.0477440357208252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,5120,0.34860798716545105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,4096,0.28540799021720886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,7168,0.47936001420021057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,3584,0.25299200415611267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,3072,0.22067199647426605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,2560,0.1892479956150055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,2048,0.1576319932937622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,1536,0.12569600343704224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,256,0.039744000881910324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,512,0.06220800057053566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,1024,0.09430400282144547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,128,0.03283200040459633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,65536,6.864287853240967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,10240,0.5769919753074646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,12288,0.6816319823265076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,8192,0.46345600485801697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,16384,0.8991680145263672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,5120,0.2999039888381958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,4096,0.24537600576877594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,7168,0.4127359986305237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,3584,0.21887999773025513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,3072,0.19155199825763702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,2560,0.1640319973230362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,2048,0.1367039978504181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,65536,5.446720123291016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,1536,0.1096000000834465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,256,0.03711999952793121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,512,0.0549440011382103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,1024,0.08204799890518188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,128,0.027583999559283257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,8192,0.3875519931316376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,10240,0.47836801409721375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,12288,0.5689280033111572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,16384,0.7518720030784607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,5120,0.25145599246025085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,4096,0.20652799308300018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,7168,0.3426240086555481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,2048,0.11574400216341019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,2560,0.13814400136470795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,3072,0.16092799603939056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,65536,4.759200096130371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,3584,0.18559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,128,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,256,0.032607998698949814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,512,0.047839999198913574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,1024,0.07011199742555618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,1536,0.09350399672985077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,8192,0.31174400448799133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,10240,0.3851200044155121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,12288,0.45664000511169434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,16384,0.6025279760360718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,5120,0.20294399559497833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,7168,0.27504000067710876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,4096,0.1666560024023056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,2048,0.09391999989748001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,2560,0.11180800199508667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,3584,0.14870400726795197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,3072,0.13017599284648895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,65536,4.200479984283447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,256,0.027871999889612198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,512,0.03920000046491623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,1536,0.07500799745321274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,1024,0.05728000029921532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,128,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,8192,0.23660799860954285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,10240,0.29180800914764404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,12288,0.3461439907550812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,16384,0.45497599244117737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,5120,0.15503999590873718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,7168,0.2094080001115799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,4096,0.12703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,2048,0.07264000177383423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,2560,0.0873280018568039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,3072,0.09964799880981445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,3584,0.11363200098276138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,65536,3.5364480018615723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,256,0.022048000246286392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,512,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,1536,0.0578560009598732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,1024,0.04339199885725975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,128,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,8192,0.16076800227165222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,10240,0.1987839937210083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,12288,0.23401600122451782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,16384,0.30636799335479736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,5120,0.10630399733781815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,4096,0.08799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,7168,0.14281600713729858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,2560,0.060864001512527466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,3072,0.06992000341415405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,3584,0.0791039988398552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,65536,2.7231040000915527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,2048,0.05164799839258194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,256,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,512,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,1024,0.03110400028526783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,128,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,1536,0.04089599847793579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,8192,0.0862400010228157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,10240,0.10425599664449692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,12288,0.12230399996042252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,16384,0.1610880047082901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,5120,0.05830400064587593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,4096,0.049247998744249344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,7168,0.0769599974155426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,65536,2.064160108566284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,2048,0.030527999624609947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,3584,0.04499199986457825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,2560,0.03558399900794029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,3072,0.040063999593257904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,1536,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,256,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,128,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,512,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,1024,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,8192,0.06035200133919716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,10240,0.07174400240182877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,12288,0.08367999643087387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,16384,0.10822399705648422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,5120,0.043807998299598694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,7168,0.05532800033688545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,4096,0.03651199862360954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,65536,1.3936959505081177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,2048,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,2560,0.026240000501275063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,1536,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,3584,0.03340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,3072,0.029632000252604485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,256,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,128,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,1024,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,512,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,8192,0.0570559985935688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,10240,0.06784000247716904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,12288,0.07971200346946716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,16384,0.10102400183677673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,65536,0.7152320146560669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,5120,0.04121600091457367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,7168,0.052000001072883606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,4096,0.034591998904943466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,1536,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,2048,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,3072,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,2560,0.022752000018954277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,3584,0.032607998698949814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,256,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,512,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,128,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,1024,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,7168,4.120128154754639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,65536,0.4398080110549927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,5120,2.9883201122283936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,65536,0.35734400153160095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,4096,2.4274239540100098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,8192,4.683775901794434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,2560,1.5892800092697144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,10240,5.807040214538574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,3072,1.8651200532913208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,3584,2.1491520404815674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,2048,1.2994240522384644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,1024,0.7360000014305115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,256,0.2764799892902374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,128,0.23270399868488312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,512,0.4551039934158325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,1536,1.016767978668213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,12288,6.942975997924805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,7168,1.0703680515289307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,10240,1.506816029548645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,8192,1.2160320281982422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,12288,1.7978880405426025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,3584,0.5619199872016907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,4096,0.6342399716377258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,3072,0.4891520142555237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,16384,2.378880023956299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,5120,0.7791360020637512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,16384,11.446847915649414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,256,0.07503999769687653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,1024,0.1980160027742386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,512,0.1255359947681427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,1536,0.27107200026512146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,2048,0.34406399726867676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,2560,0.41622400283813477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,128,0.06028800085186958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,7168,0.8076480031013489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,8192,0.9153919816017151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,10240,1.133344054222107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,12288,1.3526079654693604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,4096,0.4795199930667877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,5120,0.5888320207595825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,3584,0.4260160028934479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,3072,0.37068799138069153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,16384,1.7875839471817017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,2048,0.26182401180267334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,2560,0.31593599915504456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,512,0.0981760025024414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,1536,0.20694400370121002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,1024,0.1526080071926117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,128,0.048128001391887665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,256,0.05849599838256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,8192,0.7650240063667297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,10240,0.9546560049057007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,12288,1.1294080018997192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,16384,1.4928319454193115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,7168,0.6742720007896423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,4096,0.40169599652290344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,5120,0.49827200174331665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,2560,0.2648960053920746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,2048,0.22121599316596985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,1536,0.17478400468826294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,3072,0.3109759986400604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,3584,0.35603201389312744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,1024,0.1289599984884262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,256,0.050303999334573746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,512,0.08399999886751175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,128,0.04047999903559685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,12288,0.9045119881629944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,10240,0.7653440237045288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,8192,0.6136959791183472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,16384,1.1955840587615967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,7168,0.5454720258712769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,5120,0.3959999978542328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,4096,0.32467201352119446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,3584,0.2869119942188263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,3072,0.2502079904079437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,2560,0.21404799818992615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,2048,0.1775359958410263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,1536,0.14179199934005737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,1024,0.10518400371074677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,512,0.06838399916887283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,128,0.03379200026392937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,256,0.040800001472234726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,65536,10.481311798095703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,10240,0.6656960248947144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,12288,0.7931519746780396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,65536,7.857247829437256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,16384,1.0495359897613525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,8192,0.5406399965286255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,5120,0.3505280017852783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,7168,0.4753600060939789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,4096,0.2844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,2560,0.18915200233459473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,3072,0.22086399793624878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,3584,0.25248000025749207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,2048,0.1571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,1024,0.09324800223112106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,65536,6.553887844085693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,1536,0.12563200294971466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,512,0.062144000083208084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,256,0.038047999143600464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,128,0.03286400064826012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,8192,0.3885760009288788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,10240,0.4792959988117218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,12288,0.569599986076355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,16384,0.7516800165176392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,65536,5.1299519538879395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,4096,0.20662400126457214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,5120,0.251583993434906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,7168,0.3423359990119934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,2560,0.1377280056476593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,1536,0.0920960009098053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,2048,0.11599999666213989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,3072,0.16038399934768677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,3584,0.1839359998703003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,1024,0.06985600292682648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,256,0.029920000582933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,512,0.04684799909591675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,128,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,10240,0.38495999574661255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,8192,0.3127039968967438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,12288,0.45840001106262207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,5120,0.20339199900627136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,7168,0.27532801032066345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,16384,0.6039999723434448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,4096,0.16684800386428833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,3072,0.13065600395202637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,3584,0.14985600113868713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,2560,0.11222399771213531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,2048,0.09446399658918381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,1536,0.07561600208282471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,256,0.02582399919629097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,512,0.03977600112557411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,1024,0.05798399820923805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,128,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,65536,4.099552154541016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,10240,0.34166398644447327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,12288,0.4060800075531006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,8192,0.2829119861125946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,7168,0.24963200092315674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,16384,0.5367680191993713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,4096,0.15532800555229187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,5120,0.18694399297237396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,3584,0.13942399621009827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,3072,0.12492799758911133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,2048,0.0936959981918335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,1024,0.057312000542879105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,1536,0.07574400305747986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,2560,0.11148799955844879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,128,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,256,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,512,0.036768000572919846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,65536,3.3089599609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,12288,0.34563198685646057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,8192,0.23683199286460876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,10240,0.2921920120716095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,4096,0.12780800461769104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,5120,0.15488000214099884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,7168,0.20950399339199066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,16384,0.45583999156951904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,65536,2.3472321033477783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,65536,2.2833919525146484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,3584,0.11475200206041336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,2048,0.07267200201749802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,2560,0.08687999844551086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,3072,0.10012800246477127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,1536,0.05958399921655655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,1024,0.04572800174355507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,512,0.03222399950027466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,256,0.02191999927163124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,128,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,7168,0.1871359944343567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,8192,0.2107200026512146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,12288,0.3060159981250763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,10240,0.25804799795150757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,5120,0.1401599943637848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,4096,0.11772800236940384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,3072,0.09539200365543365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,16384,0.400160014629364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,3584,0.10768000036478043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,2560,0.08569599688053131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,512,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,2048,0.07379200309515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,1024,0.045343998819589615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,1536,0.059039998799562454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,256,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,128,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,8192,0.16128000617027283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,12288,0.23443199694156647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,10240,0.19760000705718994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,16384,0.3070400059223175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,4096,0.08851200342178345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,5120,0.10684800148010254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,7168,0.14291200041770935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,2560,0.06128000095486641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,3584,0.07971200346946716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,3072,0.07020799815654755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,65536,1.8376959562301636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,2048,0.052191998809576035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,512,0.02332800067961216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,1024,0.03388800099492073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,256,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,1536,0.043168000876903534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,128,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,8192,0.13116799294948578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,10240,0.15839999914169312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,12288,0.1841599941253662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,16384,0.23766399919986725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,4096,0.07849600166082382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,5120,0.0923520028591156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,7168,0.11872000247240067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,65536,1.6715199947357178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,3072,0.06400000303983688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,3584,0.07180800288915634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,2048,0.050144001841545105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,2560,0.057631999254226685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,256,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,1536,0.04118400067090988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,128,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,512,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,1024,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,8192,0.08476799726486206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,10240,0.10335999727249146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,12288,0.12134400010108948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,65536,1.2560319900512695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,16384,0.1584639996290207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,4096,0.0480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,5120,0.0578560009598732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,7168,0.07552000135183334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,3584,0.043807998299598694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,3072,0.03984000161290169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,1536,0.0261439997702837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,2048,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,1024,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,2560,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,512,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,256,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,128,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,8192,0.052671998739242554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,10240,0.06255999952554703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,12288,0.07280000299215317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,65536,0.8703680038452148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,16384,0.09151999652385712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,4096,0.030688000842928886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,5120,0.03814399987459183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,7168,0.04726399853825569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,3584,0.027744000777602196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,3072,0.02502400055527687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,1536,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,2560,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,2048,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,1024,0.015039999969303608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,512,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,256,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,128,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,65536,0.5931839942932129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,8192,0.04009599983692169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,12288,0.05392000079154968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,10240,0.047807998955249786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,7168,0.03667199984192848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,16384,0.06787200272083282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,4096,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,5120,0.02985600009560585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,3584,0.022655999287962914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,3072,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,2560,0.01881599985063076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,1024,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,2048,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,1536,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,512,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,256,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,128,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,65536,0.3227519989013672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,8192,0.03747199848294258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,12288,0.050655998289585114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,10240,0.04479999840259552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,16384,0.06464000046253204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,3584,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,5120,0.02800000086426735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,4096,0.023744000121951103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,7168,0.03340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,3072,0.01945599913597107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,1536,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,2048,0.01651199907064438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,2560,0.017983999103307724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,1024,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,512,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,256,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,65536,0.22864000499248505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,8192,2.352735996246338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,4096,1.2254079580307007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,65536,0.21062399446964264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,7168,2.07094407081604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,5120,1.5083520412445068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,2048,0.6634560227394104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,3072,0.9427199959754944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,10240,2.915424108505249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,2560,0.8036479949951172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,3584,1.084671974182129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,1536,0.5178239941596985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,128,0.1196800023317337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,256,0.1348479986190796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,512,0.23712000250816345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,1024,0.3776639997959137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,12288,3.4800000190734863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,8192,0.6135039925575256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,7168,0.5412799715995789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,10240,0.7586879730224609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,12288,0.9053760170936584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,5120,0.3960320055484772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,2560,0.2133760005235672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,16384,4.608543872833252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,4096,0.32233598828315735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,3584,0.2863680124282837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,3072,0.25088000297546387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,1024,0.10473600029945374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,16384,1.1953279972076416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,1536,0.14054399728775024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,512,0.06825599819421768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,256,0.041919998824596405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,2048,0.177279993891716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,128,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,8192,0.4638400077819824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,7168,0.413536012172699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,10240,0.5723519921302795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,5120,0.2998400032520294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,12288,0.6812800168991089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,4096,0.24780799448490143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,3072,0.1913599967956543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,2560,0.16364799439907074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,3584,0.21849599480628967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,16384,0.9001280069351196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,1536,0.10956799983978271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,2048,0.13580800592899323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,1024,0.0814720019698143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,512,0.055456001311540604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,256,0.03280000016093254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,128,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,8192,0.3882560133934021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,7168,0.34323200583457947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,10240,0.47891199588775635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,12288,0.5699840188026428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,5120,0.2512640058994293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,16384,0.768127977848053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,4096,0.2059520035982132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,3584,0.18377600610256195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,3072,0.16073599457740784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,2560,0.13820800185203552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,1536,0.09318400174379349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,2048,0.11478400230407715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,1024,0.06921599805355072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,256,0.02879999950528145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,512,0.04451199993491173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,128,0.02304000034928322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,16384,0.6035839915275574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,10240,0.3845759928226471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,12288,0.45763200521469116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,5120,0.20374399423599243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,8192,0.312608003616333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,7168,0.27616000175476074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,65536,5.180736064910889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,4096,0.1664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,3584,0.14931200444698334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,3072,0.1302720010280609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,2560,0.1125440001487732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,2048,0.09484799951314926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,1024,0.057312000542879105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,512,0.0382080003619194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,1536,0.07648000121116638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,256,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,128,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,65536,2.346496105194092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,65536,3.8474879264831543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,12288,0.4018239974975586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,10240,0.34012800455093384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,16384,0.5275200009346008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,7168,0.2492160052061081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,5120,0.18611200153827667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,8192,0.278656005859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,4096,0.15529599785804749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,3584,0.13996799290180206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,3072,0.12396799772977829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,1536,0.07577600330114365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,2048,0.09196799993515015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,1024,0.058079998940229416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,2560,0.10940799862146378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,512,0.03455999866127968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,256,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,65536,3.1207358837127686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,128,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,8192,0.20572799444198608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,7168,0.18345600366592407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,12288,0.2943040132522583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,10240,0.25011199712753296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,5120,0.14009599387645721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,3584,0.10675200074911118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,4096,0.1178240031003952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,16384,0.3829120099544525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,3072,0.09600000083446503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,65536,2.0356481075286865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,1024,0.04505600035190582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,2560,0.08406399935483932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,2048,0.07235199958086014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,1536,0.059167999774217606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,512,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,256,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,128,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,7168,0.14291200041770935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,8192,0.16172799468040466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,10240,0.1966720074415207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,12288,0.23369599878787994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,5120,0.10678400099277496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,16384,0.30687999725341797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,4096,0.08790399879217148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,3584,0.07840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,3072,0.06995200365781784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,2560,0.06080000102519989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,512,0.02271999977529049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,1024,0.0331839993596077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,1536,0.04265600070357323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,2048,0.05222399905323982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,256,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,128,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,8192,0.16089600324630737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,12288,0.23395200073719025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,10240,0.1974720060825348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,16384,0.3068160116672516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,4096,0.08796799927949905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,5120,0.10684800148010254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,7168,0.14246399700641632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,3584,0.07955200225114822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,2560,0.06095999851822853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,3072,0.06982400268316269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,2048,0.05167999863624573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,65536,1.4842239618301392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,1536,0.04278400167822838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,1024,0.03372799977660179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,512,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,256,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,128,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,10240,0.1579200029373169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,8192,0.13148799538612366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,12288,0.1844480037689209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,16384,0.23852799832820892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,65536,1.178655982017517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,5120,0.09232000261545181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,3584,0.07184000313282013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,7168,0.11913599818944931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,4096,0.07791999727487564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,3072,0.06505600363016129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,1024,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,512,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,1536,0.041120000183582306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,2560,0.05833600088953972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,2048,0.04947200044989586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,256,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,128,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,8192,0.11430399864912033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,12288,0.15833599865436554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,65536,1.1789439916610718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,10240,0.13580800592899323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,16384,0.20255999267101288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,3584,0.062272001057863235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,7168,0.10255999863147736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,4096,0.06844799965620041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,5120,0.08051200211048126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,65536,0.879040002822876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,1024,0.02723200060427189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,2048,0.04399999976158142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,3072,0.057472001761198044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,1536,0.03996799886226654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,2560,0.05011200159788132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,512,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,128,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,256,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,12288,0.12176000326871872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,7168,0.07673600316047668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,8192,0.08502399921417236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,10240,0.10361599922180176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,16384,0.15756799280643463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,3584,0.04476799815893173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,4096,0.04886399954557419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,5120,0.05843200162053108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,3072,0.040031999349594116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,2560,0.03526400029659271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,1536,0.0261439997702837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,2048,0.030400000512599945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,512,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,1024,0.021023999899625778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,256,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,128,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,12288,0.1202239990234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,8192,0.08537600189447403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,10240,0.10409600287675858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,65536,0.731872022151947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,16384,0.15404799580574036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,4096,0.04851200059056282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,5120,0.058079998940229416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,7168,0.07612799853086472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,3584,0.04435199871659279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,3072,0.039712000638246536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,1536,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,2560,0.0344959981739521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,2048,0.030079999938607216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,512,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,1024,0.021023999899625778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,128,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,256,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,12288,0.07711999863386154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,65536,0.5946559906005859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,10240,0.06809599697589874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,16384,0.09574399888515472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,8192,0.05689600110054016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,5120,0.04044799879193306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,7168,0.05180799961090088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,3072,0.028543999418616295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,3584,0.03097599931061268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,4096,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,2560,0.02579200081527233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,512,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,1024,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,2048,0.022336000576615334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,1536,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,256,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,128,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,65536,0.5557760000228882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,12288,0.0525440014898777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,8192,0.03836800158023834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,10240,0.04678399860858917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,16384,0.06284800171852112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,7168,0.03465599939227104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,5120,0.027807999402284622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,4096,0.024159999564290047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,3072,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,3584,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,2560,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,2048,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,1536,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,1024,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,512,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,65536,0.3516159951686859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,8192,0.034432001411914825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,12288,0.04041599854826927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,10240,0.03868800029158592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,16384,0.047520000487565994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,7168,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,5120,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,4096,0.023072000592947006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,3584,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,3072,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,2560,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,1536,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,2048,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,1024,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,512,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,65536,0.2017280012369156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,12288,0.03673600032925606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,8192,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,16384,0.04438399896025658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,10240,0.03267199918627739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,7168,0.028960000723600388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,5120,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,4096,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,3584,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,2560,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,3072,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,2048,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,512,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,1536,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,1024,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,65536,0.1372160017490387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,5120,0.7784640192985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,8192,1.2168320417404175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,7168,1.070304036140442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,3584,0.5626879930496216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,65536,0.11958400160074234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,3072,0.493120014667511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,4096,0.6410560011863708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,2560,0.4172160029411316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,2048,0.3439039885997772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,1024,0.19782400131225586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,1536,0.27344000339508057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,512,0.11478400230407715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,256,0.07574400305747986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,128,0.062272001057863235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,10240,1.5058879852294922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,12288,1.7958400249481201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,10240,0.3837440013885498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,16384,0.6023039817810059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,8192,0.3121280074119568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,12288,0.45737600326538086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,5120,0.2022079974412918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,4096,0.16592000424861908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,7168,0.2751680016517639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,2560,0.11129599809646606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,3584,0.14790399372577667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,2048,0.0931520015001297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,3072,0.13014400005340576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,1536,0.07497599720954895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,16384,2.376960039138794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,512,0.03478400036692619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,256,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,1024,0.054048001766204834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,128,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,7168,0.20895999670028687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,8192,0.23590399324893951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,10240,0.2905600070953369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,12288,0.3460479974746704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,5120,0.1544959992170334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,4096,0.1265600025653839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,16384,0.4537599980831146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,2560,0.08646400272846222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,3072,0.10108800232410431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,3584,0.11353600025177002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,512,0.029152000322937965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,1536,0.05881600081920624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,256,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,1024,0.044256001710891724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,2048,0.07248000055551529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,128,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,12288,0.2932479977607727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,10240,0.2500480115413666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,8192,0.20633600652217865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,4096,0.11807999759912491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,7168,0.18326400220394135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,5120,0.1406400054693222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,16384,0.382176011800766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,3072,0.08662399649620056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,3584,0.10729599744081497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,2560,0.07532799988985062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,2048,0.06224000081419945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,1024,0.03843199834227562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,1536,0.04995200037956238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,512,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,256,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,128,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,12288,0.23318399488925934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,65536,2.346143960952759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,10240,0.19708800315856934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,8192,0.16044799983501434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,16384,0.30559998750686646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,7168,0.14348800480365753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,5120,0.10665600001811981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,4096,0.08745600283145905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,3584,0.0783040001988411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,65536,1.761855959892273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,3072,0.06988800317049026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,2048,0.05135999992489815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,2560,0.0607680007815361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,512,0.020864000543951988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,1536,0.04259200021624565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,1024,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,256,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,128,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,8192,0.1603199988603592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,12288,0.23289600014686584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,10240,0.19699199497699738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,65536,1.4541759490966797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,16384,0.3056640028953552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,7168,0.14233599603176117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,5120,0.10672000050544739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,4096,0.08761599659919739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,3584,0.07900799810886383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,3072,0.06908799707889557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,2560,0.060736000537872314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,2048,0.05071999877691269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,1536,0.04233599826693535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,1024,0.03280000016093254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,256,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,512,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,128,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,65536,1.178879976272583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,10240,0.1356479972600937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,8192,0.11267200112342834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,12288,0.1576319932937622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,5120,0.07971200346946716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,16384,0.20153599977493286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,4096,0.06694400310516357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,3584,0.061983998864889145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,7168,0.10204800218343735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,3072,0.05644800141453743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,2560,0.04870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,2048,0.04105599969625473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,1024,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,1536,0.033984001725912094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,256,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,128,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,512,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,16384,0.15744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,8192,0.08560000360012054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,12288,0.12118399888277054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,65536,1.1775360107421875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,10240,0.10345599800348282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,4096,0.04809600114822388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,3072,0.039455998688936234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,3584,0.04419200122356415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,5120,0.0586559996008873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,7168,0.07593599706888199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,2048,0.030368000268936157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,1536,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,2560,0.03494400158524513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,1024,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,512,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,128,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,256,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,65536,0.7308480143547058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,12288,0.12147200107574463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,8192,0.08566399663686752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,10240,0.1035199984908104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,16384,0.1568319946527481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,3584,0.04479999840259552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,7168,0.07587199658155441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,3072,0.039712000638246536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,4096,0.048448000103235245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,5120,0.05798399820923805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,2048,0.030112000182271004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,1536,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,1024,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,512,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,2560,0.035071998834609985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,256,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,128,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,65536,0.5935999751091003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,7168,0.07542400062084198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,8192,0.08489599823951721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,10240,0.10412800312042236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,12288,0.11958400160074234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,16384,0.1533759981393814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,5120,0.05673599988222122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,3584,0.04390399903059006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,3072,0.039103999733924866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,2560,0.034591998904943466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,4096,0.048576001077890396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,2048,0.0297279991209507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,1536,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,256,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,1024,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,512,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,128,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,65536,0.5934720039367676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,12288,0.10294400155544281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,10240,0.08879999816417694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,8192,0.07465600222349167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,7168,0.06787200272083282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,16384,0.1321599930524826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,3584,0.04227200150489807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,5120,0.05289600044488907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,4096,0.04604800045490265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,3072,0.039264000952243805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,2560,0.034432001411914825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,512,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,1536,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,1024,0.020864000543951988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,2048,0.03001599945127964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,256,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,128,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,65536,0.5560960173606873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,12288,0.06812799721956253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,8192,0.049056001007556915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,10240,0.060896001756191254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,16384,0.08873599767684937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,7168,0.04435199871659279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,5120,0.03446400165557861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,4096,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,3584,0.0272000003606081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,65536,0.46566399931907654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,2560,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,3072,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,2048,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,512,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,1536,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,1024,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,256,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,128,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,12288,0.0663679987192154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,7168,0.043935999274253845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,10240,0.05846399813890457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,8192,0.048448000103235245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,16384,0.07887999713420868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,5120,0.03404799848794937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,3584,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,4096,0.029120000079274178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,3072,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,2560,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,65536,0.3255999982357025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,2048,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,1536,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,1024,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,512,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,256,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,8192,0.04188799858093262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,12288,0.05158400163054466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,7168,0.03763199970126152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,10240,0.04678399860858917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,16384,0.06339199841022491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,3584,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,5120,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,4096,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,3072,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,2560,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,1536,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,2048,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,1024,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,512,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,128,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,65536,0.23875199258327484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,12288,0.03999999910593033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,10240,0.03638400137424469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,8192,0.03407999873161316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,7168,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,16384,0.04473600164055824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,5120,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,3584,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,4096,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,3072,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,2560,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,2048,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,65536,0.20095999538898468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,1536,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,1024,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,256,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,12288,0.03190400078892708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,7168,0.02703999914228916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,8192,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,10240,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,16384,0.03747199848294258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,5120,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,4096,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,3584,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,65536,0.12636800110340118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,3072,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,2560,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,1024,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,1536,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,2048,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,12288,0.029983999207615852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,8192,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,7168,0.024159999564290047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,10240,0.027871999889612198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,16384,0.0331839993596077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,5120,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,3584,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,65536,0.09324800223112106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,4096,0.021023999899625778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,2560,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,3072,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,2048,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,1024,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,1536,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,8192,0.9167360067367554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,12288,1.3525760173797607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,10240,1.133504033088684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,65536,0.07791999727487564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,2560,0.3176960051059723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,16384,1.787551999092102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,5120,0.5898560285568237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,3072,0.3741439878940582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,7168,0.8069120049476624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,1024,0.1534080058336258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,4096,0.47974398732185364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,3584,0.4256640076637268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,1536,0.21094399690628052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,2048,0.26425600051879883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,128,0.049536000937223434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,256,0.06099199876189232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,512,0.1889919936656952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,7168,0.2088959962129593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,5120,0.15539200603961945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,8192,0.23695999383926392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,10240,0.291456013917923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,12288,0.348224014043808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,3584,0.1143679991364479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,4096,0.12716799974441528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,3072,0.0997759997844696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,1024,0.04524800181388855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,2048,0.07318399846553802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,2560,0.08687999844551086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,16384,0.45446398854255676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,1536,0.05990400165319443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,512,0.02956799976527691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,128,0.017503999173641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,256,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,12288,0.26659199595451355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,10240,0.22713600099086761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,8192,0.1876479983329773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,16384,0.34598401188850403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,5120,0.1281919926404953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,7168,0.16767999529838562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,3584,0.09590400010347366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,4096,0.10672000050544739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,65536,1.7620480060577393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,3072,0.08697599917650223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,1536,0.056543998420238495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,2560,0.07711999863386154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,2048,0.0647680014371872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,1024,0.0469760000705719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,512,0.025887999683618546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,128,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,256,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,12288,0.23423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,7168,0.14348800480365753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,8192,0.3004480004310608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,4096,0.08892799913883209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,5120,0.10707200318574905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,10240,0.19766399264335632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,3072,0.17574399709701538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,2560,0.09715200215578079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,3584,0.20339199900627136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,16384,0.3073279857635498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,65536,1.2960000038146973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,2048,0.12035199999809265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,1536,0.09279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,512,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,1024,0.06454399973154068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,128,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,256,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,7168,0.11964800208806992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,8192,0.13257600367069244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,12288,0.19209599494934082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,10240,0.16204799711704254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,5120,0.09296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,16384,0.2422720044851303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,3584,0.0716480016708374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,3072,0.06460800021886826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,4096,0.07945600152015686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,1536,0.044064000248909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,512,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,2048,0.05097600072622299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,1024,0.026976000517606735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,2560,0.057472001761198044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,256,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,128,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,12288,0.16892799735069275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,10240,0.1412159949541092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,8192,0.11788800358772278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,7168,0.1069440022110939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,16384,0.21065600216388702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,5120,0.08352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,4096,0.08931200206279755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,3584,0.06467200070619583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,3072,0.06108799949288368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,2560,0.05241600051522255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,2048,0.04652800038456917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,1024,0.044895999133586884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,1536,0.0671359971165657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,512,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,65536,1.1804159879684448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,256,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,128,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,12288,0.12169600278139114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,65536,0.9207680225372314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,10240,0.18083199858665466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,8192,0.15241600573062897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,16384,0.15779200196266174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,7168,0.07631999999284744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,3584,0.04419200122356415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,5120,0.05769599974155426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,4096,0.04873599857091904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,3072,0.03948799893260002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,2560,0.0777600035071373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,2048,0.06755200028419495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,1536,0.050912000238895416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,1024,0.03590400144457817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,512,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,256,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,128,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,12288,0.1207680031657219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,65536,0.7650240063667297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,8192,0.08460800349712372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,10240,0.10275200009346008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,65536,0.5942400097846985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,16384,0.15807999670505524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,7168,0.07561600208282471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,5120,0.05737600103020668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,3584,0.04387199878692627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,3072,0.038784001022577286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,4096,0.04831999912858009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,2560,0.040991999208927155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,2048,0.029952000826597214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,1536,0.025472000241279602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,1024,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,512,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,256,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,128,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,12288,0.12198399752378464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,8192,0.08473599702119827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,7168,0.07580800354480743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,16384,0.15731200575828552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,10240,0.10278400033712387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,5120,0.05718399956822395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,3584,0.04800000041723251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,65536,0.5931839942932129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,4096,0.048128001391887665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,3072,0.03977600112557411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,2560,0.03497600182890892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,1536,0.02582399919629097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,512,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,1024,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,2048,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,256,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,128,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,8192,0.08515200018882751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,12288,0.09372799843549728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,7168,0.07577600330114365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,10240,0.08102399855852127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,16384,0.1196800023317337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,5120,0.05737600103020668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,3584,0.044096000492572784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,4096,0.04902400076389313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,65536,0.5931839942932129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,3072,0.03939199820160866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,2560,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,2048,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,1536,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,512,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,1024,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,256,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,128,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,7168,0.05276799947023392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,8192,0.05801599845290184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,12288,0.0817599967122078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,10240,0.06998399645090103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,16384,0.10543999820947647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,4096,0.036928001791238785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,3072,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,65536,0.42239999771118164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,5120,0.040352001786231995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,3584,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,2560,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,1536,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,512,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,1024,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,256,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,2048,0.030047999694943428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,128,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,8192,0.08406399935483932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,10240,0.1093439981341362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,12288,0.06889600306749344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,7168,0.044064000248909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,16384,0.17414399981498718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,4096,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,3584,0.026784000918269157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,65536,0.390175998210907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,5120,0.03407999873161316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,3072,0.024320000782608986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,2560,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,2048,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,1024,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,512,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,1536,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,256,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,128,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,12288,0.05897599831223488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,8192,0.04838399961590767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,10240,0.05478399991989136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,7168,0.043776001781225204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,16384,0.0682239979505539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,4096,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,5120,0.03392000123858452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,3584,0.026655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,3072,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,2560,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,1024,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,1536,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,512,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,2048,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,256,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,65536,0.24048000574111938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,128,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,12288,0.045184001326560974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,7168,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,8192,0.03824000060558319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,10240,0.04108799993991852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,16384,0.053568001836538315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,65536,0.18883199989795685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,5120,0.02703999914228916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,3584,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,3072,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,4096,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,2560,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,2048,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,512,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,1024,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,1536,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,256,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,128,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,12288,0.038816001266241074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,8192,0.034623999148607254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,10240,0.03587200120091438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,7168,0.030719999223947525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,16384,0.04447999969124794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,5120,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,4096,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,65536,0.15625600516796112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,3072,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,3584,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,2560,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,2048,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,1536,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,1024,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,12288,0.0318400003015995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,10240,0.03190400078892708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,8192,0.02816000021994114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,7168,0.02735999971628189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,16384,0.03577600046992302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,4096,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,3072,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,5120,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,3584,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,2560,0.01651199907064438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,65536,0.09743999689817429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,2048,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,1024,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,1536,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,8192,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,12288,0.02816000021994114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,7168,0.024159999564290047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,10240,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,16384,0.031488001346588135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,65536,0.08230400085449219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,4096,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,3584,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,5120,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,3072,0.01788800023496151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,2560,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,2048,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,1536,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,7168,0.5404800176620483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,65536,0.06572800129652023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,12288,0.9040319919586182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,5120,0.39612799882888794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,4096,0.32233598828315735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,3584,0.285504013299942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,16384,1.194432020187378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,2560,0.21267199516296387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,2048,0.1780800074338913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,8192,0.612671971321106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,3072,0.25046399235725403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,256,0.04201599955558777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,1024,0.10044799745082855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,1536,0.14035199582576752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,512,0.06063999980688095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,128,0.035551998764276505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,10240,0.7583360075950623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,8192,0.16051200032234192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,7168,0.14262400567531586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,5120,0.10630399733781815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,12288,0.23308800160884857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,4096,0.08755200356245041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,10240,0.19791999459266663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,3584,0.07913599908351898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,16384,0.3059200048446655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,1536,0.04262400045990944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,2048,0.05135999992489815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,3072,0.06998399645090103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,2560,0.06054399907588959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,512,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,256,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,1024,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,128,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,12288,0.1852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,8192,0.13020800054073334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,7168,0.12336000055074692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,10240,0.16105599701404572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,16384,0.23680000007152557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,4096,0.06937599927186966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,5120,0.09385599941015244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,3072,0.05612799897789955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,2560,0.04873599857091904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,3584,0.0639680027961731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,2048,0.042399998754262924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,512,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,1024,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,1536,0.033824000507593155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,256,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,128,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,12288,0.15760000050067902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,10240,0.13468800485134125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,8192,0.11343999952077866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,16384,0.20131200551986694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,7168,0.1024319976568222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,4096,0.06803199648857117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,5120,0.07955200225114822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,65536,0.8712959885597229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,3584,0.061824001371860504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,3072,0.056223999708890915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,2560,0.047680001705884933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,2048,0.04163200035691261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,1024,0.02598400041460991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,1536,0.03392000123858452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,512,0.017920000478625298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,256,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,128,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,65536,1.1784000396728516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,10240,0.1034879982471466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,12288,0.12134400010108948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,8192,0.08511999994516373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,65536,0.7295680046081543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,16384,0.15811200439929962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,5120,0.05804799869656563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,4096,0.04870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,7168,0.07631999999284744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,3584,0.044096000492572784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,3072,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,2560,0.035711999982595444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,1536,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,2048,0.030751999467611313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,1024,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,512,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,256,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,128,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,12288,0.12147200107574463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,7168,0.07548800110816956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,16384,0.15747199952602386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,8192,0.08499199897050858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,10240,0.1032319962978363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,5120,0.05788800120353699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,3584,0.04428799822926521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,4096,0.04854400083422661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,3072,0.039583999663591385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,2560,0.035232000052928925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,2048,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,1536,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,1024,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,512,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,256,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,128,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,12288,0.10326399654150009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,65536,0.5935680270195007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,8192,0.0751039981842041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,10240,0.08937600255012512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,16384,0.13174399733543396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,7168,0.0674239993095398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,4096,0.04540799930691719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,3584,0.042047999799251556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,5120,0.05344000086188316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,3072,0.03977600112557411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,2560,0.03471999987959862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,2048,0.0297279991209507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,1536,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,1024,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,512,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,256,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,128,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,65536,0.5938559770584106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,12288,0.0679360032081604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,8192,0.04915200173854828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,10240,0.05926400050520897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,16384,0.08777599781751633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,7168,0.04527999833226204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,5120,0.03590400144457817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,3072,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,3584,0.0272000003606081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,4096,0.029920000582933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,2560,0.023231999948620796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,2048,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,512,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,1536,0.01759999990463257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,1024,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,256,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,65536,0.4670720100402832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,128,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,12288,0.06755200028419495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,7168,0.04387199878692627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,8192,0.04870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,10240,0.05897599831223488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,16384,0.08771199733018875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,4096,0.029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,5120,0.03465599939227104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,3584,0.02723200060427189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,3072,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,2560,0.022431999444961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,1536,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,2048,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,1024,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,512,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,65536,0.3314239978790283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,256,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,128,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,12288,0.06614399701356888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,10240,0.05820799991488457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,8192,0.04873599857091904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,7168,0.04342399910092354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,16384,0.07849600166082382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,5120,0.03446400165557861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,4096,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,3584,0.026944000273942947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,3072,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,2560,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,2048,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,1536,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,512,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,65536,0.30713599920272827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,1024,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,256,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,12288,0.06099199876189232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,10240,0.05398400127887726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,8192,0.04822399839758873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,7168,0.043935999274253845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,65536,0.23897600173950195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,16384,0.07353600114583969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,5120,0.033824000507593155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,3584,0.02691200003027916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,3072,0.024992000311613083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,4096,0.029343999922275543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,2560,0.02191999927163124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,2048,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,1536,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,512,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,256,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,128,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,1024,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,8192,0.03824000060558319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,12288,0.05251200124621391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,7168,0.03494400158524513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,10240,0.047231998294591904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,16384,0.06326399743556976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,65536,0.25071999430656433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,5120,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,3584,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,4096,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,3072,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,2560,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,2048,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,1536,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,512,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,1024,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,12288,0.04527999833226204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,10240,0.04057599976658821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,8192,0.03753599897027016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,65536,0.20137600600719452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,7168,0.034752000123262405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,16384,0.053119998425245285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,5120,0.027168000116944313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,3072,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,3584,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,2560,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,4096,0.023711999878287315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,2048,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,1024,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,1536,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,512,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,256,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,12288,0.03948799893260002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,8192,0.03356799855828285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,10240,0.036159999668598175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,7168,0.031808000057935715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,16384,0.044895999133586884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,65536,0.1425279974937439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,5120,0.02659199945628643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,4096,0.02287999913096428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,3584,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,2560,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,3072,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,2048,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,1536,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,1024,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,512,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,12288,0.03219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,8192,0.028831999748945236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,7168,0.026784000918269157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,10240,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,16384,0.037376001477241516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,3584,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,5120,0.02332800067961216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,4096,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,65536,0.13753600418567657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,3072,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,2560,0.016767999157309532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,1536,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,2048,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,12288,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,8192,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,7168,0.024191999807953835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,10240,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,16384,0.033535998314619064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,5120,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,3584,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,4096,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,65536,0.09068799763917923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,2560,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,3072,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,2048,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,1536,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,128,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,12288,0.02735999971628189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,7168,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,10240,0.025567999109625816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,16384,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,8192,0.023744000121951103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,3584,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,65536,0.06560000032186508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,5120,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,4096,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,2560,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,3072,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,1536,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,2048,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,256,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,65536,0.05417599901556969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,7168,0.6964160203933716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,10240,0.9672639966011047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,12288,1.1894400119781494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,4096,0.44204801321029663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,5120,0.5247039794921875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,2048,0.20803199708461761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,3072,0.3171519935131073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,2560,0.2539199888706207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,1024,0.1921599954366684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,3584,0.36111998558044434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,512,0.09523200243711472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,8192,0.8090879917144775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,1536,0.1579200029373169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,128,0.03145600110292435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,256,0.0504320003092289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,12288,0.23296000063419342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,7168,0.15875199437141418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,8192,0.18275199830532074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,5120,0.11737599968910217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,10240,0.2519040107727051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,3584,0.09839999675750732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,4096,0.09609600156545639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,3072,0.07497599720954895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,16384,0.3612160086631775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,2048,0.056703999638557434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,2560,0.06505600363016129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,1536,0.07529599964618683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,1024,0.0480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,512,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,256,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,128,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,16384,1.5641920566558838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,16384,0.31190401315689087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,10240,0.2186560034751892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,7168,0.23827199637889862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,8192,0.2670080065727234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,12288,0.25945600867271423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,3584,0.1297920048236847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,4096,0.1372160017490387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,3072,0.11110399663448334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,5120,0.16819199919700623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,2560,0.08934400230646133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,1536,0.05673599988222122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,512,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,1024,0.036896001547575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,2048,0.07417599856853485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,256,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,128,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,12288,0.2173759937286377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,10240,0.17795200645923615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,8192,0.1589439958333969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,5120,0.09772799909114838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,16384,0.2759360074996948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,7168,0.13353599607944489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,4096,0.0880960002541542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,3584,0.0777600035071373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,3072,0.06777600198984146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,2560,0.07958400249481201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,2048,0.06345599889755249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,1024,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,512,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,65536,1.4715520143508911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,1536,0.05104000121355057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,256,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,128,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,12288,0.14902399480342865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,10240,0.11462400108575821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,65536,0.9726399779319763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,8192,0.08540800213813782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,16384,0.15744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,65536,1.1932480335235596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,4096,0.0488319993019104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,3584,0.04451199993491173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,7168,0.07638400048017502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,5120,0.07049600034952164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,2560,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,3072,0.03964800015091896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,1536,0.03999999910593033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,2048,0.051072001457214355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,512,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,1024,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,256,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,128,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,10240,0.10361599922180176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,7168,0.0756480023264885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,12288,0.12144000083208084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,8192,0.084927998483181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,65536,0.8242560029029846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,16384,0.15756799280643463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,3584,0.04396799951791763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,5120,0.05756799876689911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,3072,0.0395519994199276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,4096,0.04841599985957146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,2048,0.030112000182271004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,2560,0.03497600182890892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,1536,0.037151999771595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,512,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,256,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,1024,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,128,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,12288,0.08153600245714188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,8192,0.08348800241947174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,7168,0.0783040001988411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,10240,0.06918399780988693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,16384,0.10422399640083313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,4096,0.05023999884724617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,5120,0.040192000567913055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,3072,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,65536,0.658240020275116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,3584,0.03222399950027466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,2560,0.03478400036692619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,1536,0.02521600015461445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,512,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,2048,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,1024,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,256,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,128,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,12288,0.13206399977207184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,10240,0.1138560026884079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,8192,0.0910400003194809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,7168,0.0814720019698143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,16384,0.16790400445461273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,65536,0.38201600313186646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,5120,0.062272001057863235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,3584,0.04800000041723251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,2560,0.033695999532938004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,3072,0.042208001017570496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,4096,0.04915200173854828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,2048,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,1024,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,512,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,1536,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,256,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,128,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,12288,0.12521600723266602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,8192,0.08905600011348724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,7168,0.08089599758386612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,10240,0.1096000000834465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,65536,0.4110400080680847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,5120,0.060256000608205795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,16384,0.106175996363163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,4096,0.04956800118088722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,3584,0.046560000628232956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,3072,0.04028800129890442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,1024,0.01692800037562847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,1536,0.022431999444961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,2560,0.033695999532938004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,2048,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,512,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,256,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,128,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,12288,0.07478400319814682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,7168,0.07283200323581696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,10240,0.0684799998998642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,8192,0.08665599673986435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,16384,0.09071999788284302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,5120,0.05552000179886818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,3584,0.04335999861359596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,4096,0.047807998955249786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,3072,0.03827200084924698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,2560,0.031808000057935715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,2048,0.030527999624609947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,1024,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,65536,0.3620159924030304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,1536,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,512,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,128,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,256,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,12288,0.06694400310516357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,8192,0.07369600236415863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,10240,0.06143999844789505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,7168,0.07110399752855301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,65536,0.3113600015640259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,16384,0.08243200182914734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,4096,0.039903998374938965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,5120,0.05251200124621391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,3584,0.03923200070858002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,2560,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,3072,0.03683200106024742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,2048,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,256,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,1536,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,1024,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,512,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,12288,0.04438399896025658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,7168,0.034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,10240,0.04028800129890442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,8192,0.03779200091958046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,16384,0.05430399999022484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,5120,0.027264000847935677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,4096,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,3584,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,3072,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,2560,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,2048,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,65536,0.23100799322128296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,1536,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,512,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,1024,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,256,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,128,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,12288,0.04124800115823746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,8192,0.0342399999499321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,7168,0.034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,10240,0.03743999823927879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,16384,0.046879999339580536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,65536,0.15488000214099884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,3584,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,5120,0.02703999914228916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,3072,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,4096,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,2560,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,256,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,2048,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,1536,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,1024,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,512,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,12288,0.03807999938726425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,7168,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,10240,0.0360959991812706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,8192,0.03497600182890892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,16384,0.044064000248909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,65536,0.17635199427604675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,4096,0.022784000262618065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,5120,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,3584,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,3072,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,2560,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,2048,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,1024,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,1536,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,12288,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,8192,0.027583999559283257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,7168,0.026944000273942947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,10240,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,16384,0.03551999852061272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,5120,0.025599999353289604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,4096,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,3072,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,3584,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,65536,0.09286399930715561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,2560,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,2048,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,1536,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,12288,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,10240,0.026623999699950218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,7168,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,8192,0.024639999493956566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,16384,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,5120,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,65536,0.09308800101280212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,3072,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,2560,0.016448000445961952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,4096,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,3584,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,1536,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,2048,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,12288,0.02844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,10240,0.024320000782608986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,7168,0.022048000246286392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,8192,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,65536,0.05548800155520439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,16384,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,5120,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,4096,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,3072,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,3584,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,2560,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,2048,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,1536,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,128,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,65536,0.0480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,8192,0.31244799494743347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,10240,0.38764798641204834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,5120,0.20444799959659576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,4096,0.16732800006866455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,3584,0.1502400040626526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,7168,0.27744001150131226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,3072,0.132192000746727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,2560,0.11209599673748016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,2048,0.09356799721717834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,1024,0.05673599988222122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,1536,0.07491199672222137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,256,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,512,0.035392001271247864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,128,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,12288,0.460640013217926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,12288,0.12185599654912949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,10240,0.1037760004401207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,8192,0.08607999980449677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,16384,0.15769599378108978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,7168,0.07689599692821503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,5120,0.057920001447200775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,4096,0.049536000937223434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,16384,0.6050879955291748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,3584,0.04396799951791763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,3072,0.039744000881910324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,2560,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,1536,0.025728000327944756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,512,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,2048,0.029952000826597214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,256,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,1024,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,128,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,12288,0.12003199756145477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,8192,0.08486399799585342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,10240,0.10396800190210342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,7168,0.07664000242948532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,5120,0.05817599967122078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,3584,0.0432640016078949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,4096,0.04870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,16384,0.15328000485897064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,3072,0.039264000952243805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,2560,0.03372799977660179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,65536,0.5943679809570312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,2048,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,1536,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,512,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,1024,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,256,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,128,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,12288,0.1029760017991066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,10240,0.08947200328111649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,16384,0.13174399733543396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,8192,0.07593599706888199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,7168,0.06806399673223495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,4096,0.045951999723911285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,65536,0.5559359788894653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,5120,0.054207999259233475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,3584,0.04118400067090988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,3072,0.0395519994199276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,2560,0.033984001725912094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,256,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,1024,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,2048,0.029184000566601753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,128,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,1536,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,512,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,12288,0.07363200187683105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,7168,0.04646399989724159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,10240,0.06387200206518173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,8192,0.05209600180387497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,5120,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,16384,0.09443199634552002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,3072,0.024992000311613083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,3584,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,4096,0.030848000198602676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,2048,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,2560,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,1536,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,512,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,1024,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,256,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,65536,0.4649919867515564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,12288,0.06988800317049026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,8192,0.049247998744249344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,16384,0.08723200112581253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,10240,0.060864001512527466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,65536,0.331743985414505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,7168,0.04543999955058098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,4096,0.02956799976527691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,5120,0.034591998904943466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,3072,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,3584,0.027583999559283257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,2560,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,2048,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,1536,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,1024,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,512,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,128,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,256,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,12288,0.06239999830722809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,10240,0.05459199845790863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,8192,0.04848000034689903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,7168,0.043616000562906265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,16384,0.07308799773454666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,65536,0.3019520044326782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,5120,0.034015998244285583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,3072,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,3584,0.02598400041460991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,4096,0.029600000008940697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,2560,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,2048,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,512,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,256,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,1024,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,128,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,1536,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,12288,0.05209600180387497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,7168,0.03494400158524513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,8192,0.03811199963092804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,10240,0.04572800174355507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,65536,0.2375040054321289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,16384,0.062144000083208084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,5120,0.027871999889612198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,4096,0.024032000452280045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,3072,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,3584,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,2048,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,2560,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,1536,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,1024,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,256,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,512,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,128,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,12288,0.048928000032901764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,10240,0.04399999976158142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,8192,0.03782400116324425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,16384,0.058687999844551086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,7168,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,65536,0.20080000162124634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,5120,0.02735999971628189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,2560,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,2048,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,3584,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,4096,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,3072,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,1536,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,512,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,1024,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,12288,0.04566400125622749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,8192,0.03654399886727333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,7168,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,10240,0.04124800115823746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,65536,0.18054400384426117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,16384,0.05407999828457832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,5120,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,4096,0.023744000121951103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,3584,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,3072,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,2560,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,2048,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,1024,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,1536,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,256,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,512,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,12288,0.04214400053024292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,10240,0.038495998829603195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,7168,0.03244800120592117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,65536,0.14451199769973755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,8192,0.03488000109791756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,16384,0.049536000937223434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,5120,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,3072,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,2048,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,2560,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,3584,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,4096,0.023615999147295952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,1536,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,1024,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,512,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,128,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,16384,0.04527999833226204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,12288,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,8192,0.03331200033426285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,7168,0.03161599859595299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,10240,0.03587200120091438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,5120,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,4096,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,65536,0.14950400590896606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,2560,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,3584,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,3072,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,1536,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,2048,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,1024,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,12288,0.03766399994492531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,65536,0.1271039992570877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,7168,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,10240,0.03363199904561043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,8192,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,16384,0.04243199899792671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,5120,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,4096,0.022784000262618065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,3584,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,3072,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,2048,0.015039999969303608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,2560,0.016767999157309532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,1024,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,1536,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,12288,0.03152000159025192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,8192,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,7168,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,10240,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,65536,0.09372799843549728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,16384,0.03673600032925606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,3072,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,5120,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,4096,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,2560,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,3584,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,2048,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,1536,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,12288,0.02908799983561039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,8192,0.024992000311613083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,10240,0.026335999369621277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,7168,0.024639999493956566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,65536,0.08681599795818329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,16384,0.03219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,4096,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,5120,0.022336000576615334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,3072,0.018079999834299088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,2560,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,2048,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,256,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,128,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,12288,0.02659199945628643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,65536,0.06940799951553345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,10240,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,8192,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,7168,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,16384,0.02921600081026554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,5120,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,4096,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,3584,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,2048,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,3072,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,2560,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,1536,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,1024,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,12288,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,8192,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,10240,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,7168,0.022336000576615334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,16384,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,65536,0.054496001452207565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,5120,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,4096,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,3584,0.018239999189972878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,2048,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,3072,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,1536,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,512,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,1024,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,128,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,65536,0.04809600114822388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,5120,0.22144000232219696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,7168,0.33580800890922546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,8192,0.3324800133705139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,4096,0.2024960070848465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,10240,0.4864319860935211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,3584,0.1599999964237213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,3072,0.156031996011734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,2560,0.12134400010108948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,512,0.04681599885225296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,1024,0.06032000109553337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,1536,0.08092799782752991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,2048,0.10339199751615524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,16384,0.7766079902648926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,128,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,256,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,12288,0.5991680026054382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,10240,0.11456000059843063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,7168,0.07676800340414047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,16384,0.19696000218391418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,12288,0.14819200336933136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,5120,0.05878400057554245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,8192,0.0958079993724823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,4096,0.049536000937223434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,3584,0.0453759990632534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,3072,0.03888000175356865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,2560,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,1536,0.03590400144457817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,2048,0.030112000182271004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,1024,0.02611199952661991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,512,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,256,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,128,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,12288,0.14579200744628906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,10240,0.11708799749612808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,7168,0.07583999633789062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,8192,0.08515200018882751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,16384,0.1818239986896515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,4096,0.049984000623226166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,3584,0.045184001326560974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,5120,0.06665600091218948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,3072,0.03862399980425835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,2560,0.03558399900794029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,512,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,256,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,2048,0.029823999851942062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,1536,0.024992000311613083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,1024,0.023264000192284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,128,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,10240,0.10521599650382996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,12288,0.11807999759912491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,16384,0.15043200552463531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,8192,0.08851200342178345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,65536,0.6321280002593994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,4096,0.05132799968123436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,7168,0.07648000121116638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,5120,0.059039998799562454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,3584,0.045504000037908554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,3072,0.0424639992415905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,65536,0.7485120296478271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,2048,0.03017600066959858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,2560,0.0352960005402565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,1536,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,1024,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,512,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,256,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,128,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,12288,0.11603199690580368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,8192,0.08409599959850311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,7168,0.07334399968385696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,16384,0.15619200468063354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,10240,0.10371199995279312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,5120,0.05923200026154518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,65536,0.6123520135879517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,4096,0.046560000628232956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,2560,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,3584,0.04294399917125702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,3072,0.03702399879693985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,2048,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,1536,0.020896000787615776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,1024,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,512,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,256,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,128,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,12288,0.11468800157308578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,8192,0.08022399991750717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,7168,0.07056000083684921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,16384,0.08796799927949905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,10240,0.09734400361776352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,65536,0.3125759959220886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,5120,0.05769599974155426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,4096,0.045184001326560974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,3584,0.039903998374938965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,2560,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,3072,0.037151999771595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,2048,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,1536,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,1024,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,256,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,512,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,128,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,12288,0.0634239986538887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,7168,0.058111999183893204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,8192,0.0687360018491745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,10240,0.05516799911856651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,16384,0.07276800274848938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,65536,0.2849920094013214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,5120,0.04604800045490265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,3072,0.03276799991726875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,3584,0.03718400001525879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,4096,0.03673600032925606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,2560,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,2048,0.023135999217629433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,1024,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,1536,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,512,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,128,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,12288,0.06095999851822853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,8192,0.05430399999022484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,7168,0.04934399947524071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,10240,0.05167999863624573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,16384,0.06963200122117996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,5120,0.037728000432252884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,4096,0.03062400035560131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,2560,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,3072,0.024447999894618988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,3584,0.026976000517606735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,1536,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,65536,0.21648000180721283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,1024,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,2048,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,512,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,128,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,12288,0.0544000007212162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,65536,0.2616960108280182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,8192,0.05167999863624573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,10240,0.047520000487565994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,16384,0.06639999896287918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,7168,0.04569600149989128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,4096,0.030592000111937523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,5120,0.03526400029659271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,3584,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,2560,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,3072,0.023711999878287315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,2048,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,1536,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,256,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,512,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,1024,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,128,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,12288,0.05097600072622299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,10240,0.04428799822926521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,8192,0.0398080013692379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,16384,0.05875200033187866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,7168,0.042367998510599136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,65536,0.22198399901390076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,5120,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,3072,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,2560,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,4096,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,3584,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,2048,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,1536,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,1024,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,256,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,512,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,128,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,12288,0.045184001326560974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,8192,0.03759999945759773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,7168,0.0344959981739521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,10240,0.03964800015091896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,16384,0.05196800082921982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,5120,0.031968001276254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,65536,0.14368000626564026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,2560,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,4096,0.02502400055527687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,3584,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,3072,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,2048,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,1536,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,128,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,256,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,512,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,65536,0.16527999937534332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,12288,0.04255999997258186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,7168,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,10240,0.03888000175356865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,8192,0.03478400036692619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,16384,0.04649600014090538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,4096,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,5120,0.02486399933695793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,3072,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,3584,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,2560,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,2048,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,1024,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,1536,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,512,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,12288,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,65536,0.12307199835777283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,16384,0.042688000947237015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,10240,0.03190400078892708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,8192,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,7168,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,5120,0.02521600015461445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,4096,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,3584,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,3072,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,2560,0.01788800023496151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,2048,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,1536,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,1024,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,12288,0.0315839983522892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,8192,0.028991999104619026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,10240,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,16384,0.035360001027584076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,7168,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,65536,0.09910400211811066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,5120,0.02486399933695793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,4096,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,3072,0.01744000054895878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,3584,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,2560,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,2048,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,1536,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,1024,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,12288,0.02723200060427189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,8192,0.02457600086927414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,16384,0.03215999901294708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,7168,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,65536,0.08079999685287476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,10240,0.026016000658273697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,5120,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,4096,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,3584,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,3072,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,2560,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,1536,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,512,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,65536,0.054496001452207565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,12288,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,16384,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,10240,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,7168,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,8192,0.022431999444961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,5120,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,3584,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,3072,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,4096,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,2048,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,2560,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,1536,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,1024,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,256,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,12288,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,7168,0.024447999894618988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,8192,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,65536,0.04588799923658371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,10240,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,16384,0.026976000517606735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,5120,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,4096,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,2560,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,3072,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,3584,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,2048,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,1536,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,65536,0.04668800160288811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,5120,0.2800639867782593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,8192,0.4350399971008301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,4096,0.21929599344730377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,12288,0.5264959931373596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,10240,0.5520960092544556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,3072,0.16128000617027283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,2560,0.1483840048313141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,7168,0.3758080005645752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,2048,0.11356800049543381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,1536,0.08803199976682663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,3584,0.1844799965620041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,16384,0.8961279988288879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,128,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,256,0.02879999950528145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,512,0.05075199902057648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,1024,0.0615679994225502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,12288,0.1700800061225891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,5120,0.07372800260782242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,8192,0.10566399991512299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,7168,0.09404800087213516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,10240,0.1398400068283081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,16384,0.2234559953212738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,3072,0.04566400125622749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,3584,0.053247999399900436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,2560,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,4096,0.05593600124120712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,2048,0.03356799855828285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,1536,0.040031999349594116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,512,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,1024,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,128,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,256,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,7168,0.09247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,8192,0.08486399799585342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,10240,0.12956799566745758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,16384,0.1913280040025711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,12288,0.15219199657440186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,3584,0.05225599929690361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,5120,0.06985600292682648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,4096,0.049215998500585556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,2560,0.04032000154256821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,3072,0.03996799886226654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,2048,0.03340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,65536,0.876479983329773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,1024,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,512,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,1536,0.026655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,256,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,128,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,10240,0.11542399972677231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,8192,0.09251199662685394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,12288,0.12889599800109863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,16384,0.1589439958333969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,7168,0.08771199733018875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,65536,0.7341439723968506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,4096,0.04864000156521797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,5120,0.06860800087451935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,2560,0.03814399987459183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,3584,0.044096000492572784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,3072,0.04156799986958504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,2048,0.03094400092959404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,1536,0.032287999987602234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,256,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,512,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,128,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,1024,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,8192,0.09120000153779984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,7168,0.07971200346946716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,10240,0.11062400043010712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,12288,0.1318719983100891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,16384,0.1693439930677414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,3584,0.0459199994802475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,5120,0.06332799792289734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,4096,0.04944000020623207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,65536,0.6174719929695129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,3072,0.040672000497579575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,2560,0.03340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,2048,0.029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,256,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,512,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,1536,0.02287999913096428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,1024,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,128,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,12288,0.13087999820709229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,8192,0.08649600297212601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,10240,0.11382400244474411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,16384,0.08825600147247314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,65536,0.33004799485206604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,7168,0.08236800134181976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,5120,0.061792001128196716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,4096,0.048608001321554184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,3584,0.04297599941492081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,3072,0.03932800143957138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,2560,0.03411199897527695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,1536,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,2048,0.029952000826597214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,1024,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,512,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,256,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,128,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,8192,0.08032000064849854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,12288,0.06467200070619583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,10240,0.05548800155520439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,16384,0.07248000055551529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,7168,0.07260800153017044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,65536,0.28012800216674805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,5120,0.0578560009598732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,3584,0.03920000046491623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,4096,0.04527999833226204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,2560,0.03219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,3072,0.03667199984192848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,2048,0.027583999559283257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,1536,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,1024,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,512,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,256,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,128,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,12288,0.05987200140953064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,7168,0.05289600044488907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,8192,0.058559998869895935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,10240,0.05222399905323982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,65536,0.2109760046005249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,16384,0.07273600250482559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,5120,0.041760001331567764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,2048,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,3584,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,4096,0.03481600061058998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,3072,0.027648000046610832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,2560,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,1536,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,1024,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,512,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,128,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,256,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,12288,0.05711999908089638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,65536,0.2552320063114166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,8192,0.05798399820923805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,7168,0.050783999264240265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,10240,0.048767998814582825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,16384,0.06771200150251389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,4096,0.03324799984693527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,5120,0.03964800015091896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,3584,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,3072,0.02723200060427189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,2560,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,2048,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,1536,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,1024,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,512,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,256,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,128,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,12288,0.050944000482559204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,65536,0.22777600586414337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,8192,0.04032000154256821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,7168,0.04995200037956238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,10240,0.044895999133586884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,16384,0.05993599817156792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,5120,0.038336001336574554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,4096,0.032607998698949814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,3584,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,2560,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,3072,0.026335999369621277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,2048,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,1536,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,1024,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,128,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,512,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,256,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,12288,0.0453759990632534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,65536,0.14422400295734406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,16384,0.05315199866890907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,10240,0.040511999279260635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,8192,0.03791999816894531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,7168,0.03494400158524513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,5120,0.03759999945759773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,4096,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,2560,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,3584,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,3072,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,2048,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,1536,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,1024,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,256,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,128,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,512,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,12288,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,65536,0.16764800250530243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,16384,0.04966399818658829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,7168,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,8192,0.03497600182890892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,10240,0.03936000168323517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,5120,0.026944000273942947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,4096,0.022975999861955643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,3584,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,2560,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,3072,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,2048,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,1536,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,1024,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,512,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,256,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,128,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,12288,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,10240,0.030719999223947525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,8192,0.028543999418616295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,7168,0.027615999802947044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,16384,0.04255999997258186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,5120,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,65536,0.14153599739074707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,4096,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,3584,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,2560,0.01775999926030636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,3072,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,2048,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,1024,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,512,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,1536,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,256,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,65536,0.11113599687814713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,12288,0.032416000962257385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,8192,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,10240,0.028831999748945236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,7168,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,16384,0.0360959991812706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,5120,0.025119999423623085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,4096,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,3584,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,3072,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,2560,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,2048,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,1536,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,512,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,128,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,12288,0.027744000777602196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,65536,0.07948800176382065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,10240,0.025919999927282333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,7168,0.023264000192284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,16384,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,8192,0.023903999477624893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,5120,0.022975999861955643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,4096,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,2560,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,3584,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,3072,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,1536,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,2048,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,512,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,256,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,128,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,12288,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,65536,0.05584000051021576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,8192,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,16384,0.03270399942994118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,7168,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,10240,0.023391999304294586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,5120,0.021023999899625778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,4096,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,3584,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,3072,0.017920000478625298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,2560,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,2048,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,1536,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,1024,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,512,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,256,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,128,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,12288,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,65536,0.0490880012512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,10240,0.026240000501275063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,8192,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,7168,0.0261439997702837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,16384,0.029791999608278275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,4096,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,5120,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,2048,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,3072,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,3584,0.019168000668287277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,2560,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,1536,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,512,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,128,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,256,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,65536,0.05337600037455559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,5120,0.14099200069904327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,7168,0.1908479928970337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,8192,0.21187199652194977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,12288,0.3083840012550354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,10240,0.2603519856929779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,4096,0.11433599889278412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,3584,0.10339199751615524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,3072,0.09151999652385712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,2048,0.06384000182151794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,2560,0.07875200361013412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,1024,0.03984000161290169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,1536,0.05305600166320801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,16384,0.40115201473236084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,256,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,128,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,512,0.028095999732613564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,12288,0.08316799998283386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,7168,0.055904000997543335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,5120,0.0424639992415905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,8192,0.05987200140953064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,10240,0.07203199714422226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,16384,0.10809600353240967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,4096,0.03651199862360954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,2560,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,1536,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,2048,0.02364799939095974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,3584,0.03299200162291527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,3072,0.030079999938607216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,1024,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,512,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,256,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,128,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,12288,0.08092799782752991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,10240,0.06921599805355072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,8192,0.057312000542879105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,7168,0.05129599943757057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,16384,0.10339199751615524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,5120,0.040800001472234726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,4096,0.03347200155258179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,3584,0.030848000198602676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,3072,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,2560,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,2048,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,1536,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,65536,0.3943359851837158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,1024,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,512,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,128,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,256,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,12288,0.08006399869918823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,8192,0.0578560009598732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,10240,0.0692799985408783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,7168,0.05135999992489815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,16384,0.10412800312042236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,65536,0.380511999130249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,5120,0.03920000046491623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,4096,0.03376000002026558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,3584,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,3072,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,2560,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,2048,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,1536,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,1024,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,256,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,512,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,128,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,12288,0.0541439987719059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,7168,0.03638400137424469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,8192,0.03948799893260002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,16384,0.06764800101518631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,10240,0.045632001012563705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,5120,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,4096,0.024000000208616257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,3072,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,3584,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,2560,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,65536,0.37964800000190735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,2048,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,1536,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,1024,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,128,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,512,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,8192,0.039264000952243805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,7168,0.03497600182890892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,12288,0.05158400163054466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,10240,0.0459199994802475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,65536,0.235167995095253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,16384,0.06265600025653839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,4096,0.023072000592947006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,5120,0.02703999914228916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,2560,0.017920000478625298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,3584,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,3072,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,2048,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,256,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,1024,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,512,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,1536,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,128,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,12288,0.043776001781225204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,7168,0.034304000437259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,8192,0.03753599897027016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,10240,0.03964800015091896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,65536,0.2149759978055954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,16384,0.05257600173354149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,5120,0.02611199952661991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,4096,0.02304000034928322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,3584,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,2560,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,3072,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,2048,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,1536,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,256,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,1024,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,512,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,12288,0.03964800015091896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,65536,0.16681599617004395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,7168,0.03299200162291527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,8192,0.034912001341581345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,16384,0.04831999912858009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,10240,0.036959998309612274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,4096,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,3584,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,5120,0.02566399984061718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,2560,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,2048,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,3072,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,1536,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,256,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,1024,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,12288,0.040672000497579575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,10240,0.036928001791238785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,7168,0.030368000268936157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,8192,0.03465599939227104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,16384,0.04601600021123886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,65536,0.142752006649971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,5120,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,4096,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,3584,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,2560,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,2048,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,3072,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,1536,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,256,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,1024,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,12288,0.038176000118255615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,65536,0.132192000746727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,7168,0.0297279991209507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,8192,0.030751999467611313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,10240,0.03455999866127968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,16384,0.044863998889923096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,5120,0.02502400055527687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,4096,0.022495999932289124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,3072,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,2560,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,3584,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,2048,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,1536,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,512,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,1024,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,12288,0.03651199862360954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,7168,0.02876799926161766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,10240,0.03174399957060814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,8192,0.029600000008940697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,16384,0.04131200164556503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,65536,0.10636799782514572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,5120,0.023744000121951103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,2560,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,3584,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,4096,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,3072,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,2048,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,1536,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,1024,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,12288,0.03190400078892708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,7168,0.026176000013947487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,8192,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,10240,0.030527999624609947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,16384,0.036639999598264694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,65536,0.10815999656915665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,5120,0.023264000192284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,4096,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,3584,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,2048,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,3072,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,1536,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,128,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,1024,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,65536,0.09513600170612335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,12288,0.03215999901294708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,16384,0.03651199862360954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,10240,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,7168,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,8192,0.027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,5120,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,3584,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,4096,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,2560,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,2048,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,65536,0.07657600194215775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,12288,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,10240,0.02739199995994568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,16384,0.03187200054526329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,8192,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,7168,0.022752000018954277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,4096,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,5120,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,3584,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,2048,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,12288,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,10240,0.02521600015461445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,65536,0.0631679967045784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,8192,0.022975999861955643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,16384,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,7168,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,5120,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,4096,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,3072,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,2560,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,2048,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,3584,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,256,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,12288,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,65536,0.05164799839258194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,16384,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,8192,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,10240,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,7168,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,5120,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,4096,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,2560,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,2048,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,3584,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,3072,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,1536,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,1024,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,256,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,65536,0.04819199815392494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,12288,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,8192,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,16384,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,7168,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,10240,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,4096,0.01926399953663349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,5120,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,3584,0.01945599913597107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,2560,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,65536,0.04182400181889534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,5120,0.14336000382900238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,4096,0.11737599968910217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,7168,0.1937599927186966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,8192,0.21651199460029602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,10240,0.2666879892349243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,12288,0.3168320059776306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,3584,0.10559999942779541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,3072,0.09312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,2560,0.08278399705886841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,2048,0.06604799628257751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,16384,0.4102399945259094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,1536,0.054336000233888626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,1024,0.041728001087903976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,256,0.019168000668287277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,512,0.028704000636935234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,128,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,12288,0.08902399986982346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,5120,0.044224001467227936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,7168,0.05488000065088272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,8192,0.06672000139951706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,10240,0.07494399696588516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,2560,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,4096,0.03577600046992302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,3584,0.033824000507593155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,16384,0.11376000195741653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,3072,0.03062400035560131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,2048,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,512,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,1024,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,1536,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,256,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,128,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,12288,0.08079999685287476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,7168,0.05142400041222572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,8192,0.058079998940229416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,16384,0.10361599922180176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,10240,0.06963200122117996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,5120,0.03964800015091896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,4096,0.03347200155258179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,3584,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,2560,0.025151999667286873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,2048,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,3072,0.02800000086426735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,65536,0.4029119908809662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,1024,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,1536,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,512,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,256,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,128,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,8192,0.05721599981188774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,7168,0.05257600173354149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,12288,0.08054400235414505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,16384,0.10396800190210342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,10240,0.06889600306749344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,4096,0.03280000016093254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,65536,0.37907201051712036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,5120,0.038816001266241074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,3584,0.03062400035560131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,3072,0.028063999488949776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,2560,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,2048,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,1536,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,1024,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,256,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,512,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,128,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,12288,0.07155200093984604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,7168,0.04521600157022476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,8192,0.052319999784231186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,10240,0.060864001512527466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,16384,0.09033600240945816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,65536,0.39347198605537415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,5120,0.03407999873161316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,3072,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,4096,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,3584,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,2560,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,1536,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,2048,0.017983999103307724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,512,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,1024,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,256,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,128,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,12288,0.05177599936723709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,8192,0.04851200059056282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,10240,0.05782400071620941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,16384,0.062144000083208084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,7168,0.044096000492572784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,5120,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,3584,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,4096,0.027648000046610832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,3072,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,2560,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,2048,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,1536,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,1024,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,256,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,512,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,65536,0.3548800051212311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,128,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,12288,0.045504000037908554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,7168,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,10240,0.03840000182390213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,8192,0.03718400001525879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,16384,0.0514880008995533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,65536,0.20611199736595154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,4096,0.023744000121951103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,5120,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,3584,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,3072,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,2048,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,2560,0.017920000478625298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,1536,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,256,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,512,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,1024,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,12288,0.040511999279260635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,7168,0.0342399999499321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,8192,0.034432001411914825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,65536,0.1732800006866455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,10240,0.03750399872660637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,16384,0.04572800174355507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,5120,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,3072,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,4096,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,3584,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,2560,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,2048,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,1536,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,1024,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,256,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,12288,0.04118400067090988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,8192,0.03455999866127968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,7168,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,10240,0.03670400008559227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,16384,0.04447999969124794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,5120,0.02768000029027462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,65536,0.1401599943637848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,4096,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,3072,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,3584,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,2560,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,2048,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,1024,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,1536,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,128,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,12288,0.03951999917626381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,8192,0.033504001796245575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,7168,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,10240,0.0344959981739521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,16384,0.04374400153756142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,65536,0.12406399846076965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,5120,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,4096,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,3072,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,3584,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,2560,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,2048,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,1024,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,1536,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,12288,0.03551999852061272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,7168,0.028095999732613564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,8192,0.03200000151991844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,65536,0.1042879968881607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,10240,0.032255999743938446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,16384,0.04057599976658821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,5120,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,3072,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,2560,0.01692800037562847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,2048,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,4096,0.023744000121951103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,3584,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,1536,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,1024,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,65536,0.09884800016880035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,12288,0.03222399950027466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,8192,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,10240,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,7168,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,16384,0.03596799820661545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,5120,0.02364799939095974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,4096,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,3072,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,3584,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,12288,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,10240,0.028896000236272812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,16384,0.035232000052928925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,8192,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,7168,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,65536,0.0926399976015091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,5120,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,4096,0.020640000700950623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,1536,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,1024,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,65536,0.07391999661922455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,12288,0.027615999802947044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,8192,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,16384,0.030688000842928886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,7168,0.026176000013947487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,10240,0.025728000327944756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,5120,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,4096,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,3072,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,3584,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,2560,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,1536,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,12288,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,65536,0.06230400130152702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,7168,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,10240,0.023296000435948372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,8192,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,16384,0.029120000079274178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,5120,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,4096,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,3584,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,1536,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,65536,0.04646399989724159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,12288,0.025472000241279602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,8192,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,10240,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,7168,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,16384,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,5120,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,4096,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,2048,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,2560,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,3584,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,128,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,12288,0.026623999699950218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,65536,0.047839999198913574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,8192,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,7168,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,10240,0.025887999683618546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,16384,0.02735999971628189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,5120,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,3584,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,2560,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,4096,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,65536,0.04320000112056732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,4096,0.13583999872207642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,5120,0.15107199549674988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,7168,0.20563200116157532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,12288,0.3293760120868683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,10240,0.2791999876499176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,3072,0.09923200309276581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,2560,0.08483199775218964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,8192,0.22809599339962006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,2048,0.07107199728488922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,1536,0.056543998420238495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,3584,0.11350400000810623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,16384,0.4275520145893097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,512,0.027807999402284622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,1024,0.042047999799251556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,128,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,256,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,5120,0.047200001776218414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,12288,0.09398400038480759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,8192,0.06665600091218948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,10240,0.08275199681520462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,7168,0.060256000608205795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,16384,0.11820799857378006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,2048,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,2560,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,4096,0.03750399872660637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,3584,0.03510399907827377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,3072,0.03222399950027466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,1536,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,512,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,256,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,128,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,1024,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,12288,0.08057600259780884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,16384,0.10335999727249146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,7168,0.05315199866890907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,8192,0.057440001517534256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,10240,0.06889600306749344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,5120,0.03920000046491623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,4096,0.033663999289274216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,3584,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,3072,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,2560,0.02595200017094612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,2048,0.02191999927163124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,1536,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,1024,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,65536,0.420991986989975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,512,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,256,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,128,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,12288,0.08009599894285202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,10240,0.06911999732255936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,7168,0.05129599943757057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,8192,0.05696000158786774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,16384,0.10326399654150009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,5120,0.03923200070858002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,65536,0.3829759955406189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,3584,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,4096,0.032607998698949814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,3072,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,2560,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,1536,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,2048,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,1024,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,256,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,512,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,128,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,12288,0.07916799932718277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,8192,0.05251200124621391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,7168,0.04739199951291084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,16384,0.09494400024414062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,10240,0.06748799979686737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,5120,0.035392001271247864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,65536,0.42182400822639465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,3584,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,3072,0.02457600086927414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,4096,0.030719999223947525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,2560,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,1536,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,2048,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,512,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,1024,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,128,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,256,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,12288,0.054655998945236206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,7168,0.04940799996256828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,8192,0.05180799961090088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,10240,0.06425599753856659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,16384,0.062111999839544296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,5120,0.03488000109791756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,65536,0.3779520094394684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,4096,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,3072,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,3584,0.026176000013947487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,2560,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,2048,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,1536,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,512,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,256,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,1024,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,12288,0.045504000037908554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,8192,0.03884800150990486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,7168,0.03392000123858452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,10240,0.03855999931693077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,65536,0.20367999374866486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,16384,0.051263999193906784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,4096,0.025472000241279602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,5120,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,3584,0.024639999493956566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,3072,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,2560,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,2048,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,1536,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,1024,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,256,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,512,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,12288,0.04201599955558777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,8192,0.03807999938726425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,7168,0.03404799848794937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,16384,0.04531199857592583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,10240,0.038047999143600464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,4096,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,5120,0.02800000086426735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,65536,0.15600000321865082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,3584,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,2560,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,3072,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,1536,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,2048,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,1024,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,256,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,128,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,12288,0.04073600098490715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,7168,0.03292800113558769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,8192,0.03590400144457817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,10240,0.03753599897027016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,16384,0.04412800073623657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,65536,0.13087999820709229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,5120,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,3072,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,2560,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,3584,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,4096,0.022784000262618065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,2048,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,1536,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,512,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,1024,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,12288,0.03951999917626381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,7168,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,8192,0.03356799855828285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,65536,0.12169600278139114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,16384,0.047168001532554626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,10240,0.038047999143600464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,5120,0.025887999683618546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,2048,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,3072,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,2560,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,3584,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,4096,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,1536,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,1024,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,12288,0.035392001271247864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,65536,0.10454399883747101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,7168,0.028672000393271446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,8192,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,10240,0.031968001276254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,16384,0.040672000497579575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,5120,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,3584,0.020640000700950623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,4096,0.024159999564290047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,3072,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,2560,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,2048,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,1536,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,1024,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,12288,0.03302399814128876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,65536,0.10278400033712387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,7168,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,8192,0.029152000322937965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,10240,0.030400000512599945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,16384,0.03766399994492531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,4096,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,5120,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,3584,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,2048,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,256,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,65536,0.09222400188446045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,12288,0.0315839983522892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,7168,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,8192,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,10240,0.02783999964594841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,16384,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,5120,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,4096,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,3072,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,3584,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,2048,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,1536,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,12288,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,65536,0.07782399654388428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,16384,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,7168,0.025599999353289604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,10240,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,8192,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,4096,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,5120,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,3584,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,3072,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,1536,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,12288,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,65536,0.06614399701356888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,10240,0.024064000695943832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,16384,0.028704000636935234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,7168,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,8192,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,5120,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,4096,0.01926399953663349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,3072,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,3584,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,2048,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,2560,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,512,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,65536,0.046592000871896744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,12288,0.027744000777602196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,16384,0.02812799997627735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,10240,0.02735999971628189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,8192,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,7168,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,4096,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,5120,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,3584,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,12288,0.02864000014960766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,65536,0.04652800038456917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,8192,0.026464000344276428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,10240,0.027744000777602196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,16384,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,7168,0.026176000013947487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,5120,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,4096,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,3072,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,3584,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,2560,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,1536,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,1024,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,128,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,65536,0.04492799937725067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,5120,0.13072000443935394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,7168,0.17923200130462646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,8192,0.20255999267101288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,12288,0.29407998919487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,10240,0.2502720057964325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,4096,0.10486400127410889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,2560,0.07142399996519089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,3072,0.08246400207281113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,3584,0.09558399766683578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,2048,0.059039998799562454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,1024,0.034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,1536,0.04572800174355507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,256,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,16384,0.3893119990825653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,128,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,512,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,12288,0.08371199667453766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,5120,0.04095999896526337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,8192,0.058079998940229416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,10240,0.07212799787521362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,7168,0.05478399991989136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,16384,0.1032319962978363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,4096,0.03516799956560135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,3584,0.03254399821162224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,2560,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,3072,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,2048,0.022911999374628067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,1536,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,1024,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,256,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,512,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,8192,0.049056001007556915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,12288,0.06774400174617767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,10240,0.057920001447200775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,7168,0.04396799951791763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,16384,0.08649600297212601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,3584,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,4096,0.028736000880599022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,2560,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,5120,0.03440000116825104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,3072,0.024032000452280045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,2048,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,65536,0.3778879940509796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,512,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,256,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,1536,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,1024,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,12288,0.0674239993095398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,8192,0.04867200180888176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,7168,0.04364800080657005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,10240,0.05814399942755699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,16384,0.08646400272846222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,4096,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,5120,0.03308799862861633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,3584,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,3072,0.023903999477624893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,2560,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,2048,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,1024,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,1536,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,512,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,256,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,65536,0.3118720054626465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,128,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,12288,0.051231998950242996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,10240,0.04390399903059006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,8192,0.03728000074625015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,7168,0.033215999603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,16384,0.06419199705123901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,65536,0.3102720081806183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,5120,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,4096,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,3584,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,3072,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,2560,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,1536,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,2048,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,1024,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,256,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,12288,0.04947200044989586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,7168,0.03315199911594391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,8192,0.037696000188589096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,16384,0.059647999703884125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,10240,0.04323200136423111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,65536,0.2120320051908493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,5120,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,3072,0.01881599985063076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,3584,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,2560,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,4096,0.02223999984562397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,2048,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,1536,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,1024,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,12288,0.039872001856565475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,8192,0.033504001796245575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,7168,0.030112000182271004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,16384,0.04774399846792221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,10240,0.037087999284267426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,65536,0.19395199418067932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,5120,0.026464000344276428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,3072,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,2560,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,4096,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,2048,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,3584,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,1536,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,1024,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,12288,0.03702399879693985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,65536,0.1425279974937439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,8192,0.030912000685930252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,7168,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,16384,0.04185599833726883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,10240,0.03299200162291527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,5120,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,4096,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,2560,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,3584,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,2048,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,3072,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,1536,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,1024,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,12288,0.0360959991812706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,7168,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,65536,0.1212799996137619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,10240,0.034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,8192,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,16384,0.04156799986958504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,5120,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,3072,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,3584,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,4096,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,1536,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,12288,0.03526400029659271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,65536,0.10979200154542923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,16384,0.03993599861860275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,7168,0.025919999927282333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,8192,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,10240,0.03126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,5120,0.02364799939095974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,4096,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,3584,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,2048,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,3072,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,1536,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,128,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,12288,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,65536,0.09622400254011154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,8192,0.028192000463604927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,7168,0.02595200017094612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,16384,0.03641600161790848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,10240,0.02940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,5120,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,4096,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,3584,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,2048,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,3072,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,1536,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,1024,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,12288,0.029120000079274178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,65536,0.08767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,8192,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,16384,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,10240,0.02768000029027462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,7168,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,5120,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,4096,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,3584,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,2560,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,2048,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,128,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,65536,0.07820799946784973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,12288,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,16384,0.030880000442266464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,7168,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,8192,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,10240,0.026944000273942947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,5120,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,4096,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,3584,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,2560,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,1536,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,12288,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,65536,0.06735999882221222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,7168,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,8192,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,10240,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,16384,0.029823999851942062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,5120,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,3584,0.017920000478625298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,4096,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,3072,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,2048,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,1536,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,12288,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,65536,0.05286400020122528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,8192,0.022752000018954277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,7168,0.022495999932289124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,10240,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,16384,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,5120,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,4096,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,2560,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,3072,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,2048,0.01375999953597784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,1536,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,65536,0.04543999955058098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,12288,0.02985600009560585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,8192,0.028736000880599022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,16384,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,10240,0.03155200183391571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,7168,0.02611199952661991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,5120,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,4096,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,3584,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,2560,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,12288,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,8192,0.02768000029027462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,16384,0.031936001032590866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,7168,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,10240,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,65536,0.04182400181889534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,5120,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,4096,0.01881599985063076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,2560,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,3072,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,2048,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,3584,0.017983999103307724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,65536,0.04368000105023384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,5120,0.1279360055923462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,4096,0.10704000294208527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,8192,0.19916799664497375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,12288,0.2964800000190735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,7168,0.1759359985589981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,3584,0.0936959981918335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,3072,0.081727996468544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,2560,0.07072000205516815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,10240,0.25014400482177734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,2048,0.0594559982419014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,1536,0.04694399982690811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,1024,0.033824000507593155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,512,0.02348800003528595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,256,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,128,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,12288,0.08115199953317642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,16384,0.38383999466896057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,8192,0.05862399935722351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,7168,0.054207999259233475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,10240,0.07184000313282013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,16384,0.10284800082445145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,4096,0.034432001411914825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,3584,0.031968001276254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,5120,0.041152000427246094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,3072,0.028224000707268715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,2560,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,2048,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,1536,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,512,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,256,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,1024,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,12288,0.06892800331115723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,8192,0.04896000027656555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,7168,0.04399999976158142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,10240,0.058400001376867294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,16384,0.08604799956083298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,5120,0.034623999148607254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,4096,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,3584,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,3072,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,2560,0.022207999601960182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,2048,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,1536,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,512,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,1024,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,256,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,65536,0.37862399220466614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,12288,0.06710399687290192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,10240,0.057440001517534256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,8192,0.047968000173568726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,7168,0.04358400031924248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,16384,0.08588799834251404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,5120,0.033663999289274216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,3072,0.02396799996495247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,65536,0.3133760094642639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,3584,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,4096,0.02879999950528145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,2560,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,256,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,2048,0.01942400075495243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,128,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,1536,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,512,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,1024,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,12288,0.0504320003092289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,7168,0.034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,8192,0.03766399994492531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,10240,0.04569600149989128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,16384,0.06425599753856659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,65536,0.3097279965877533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,5120,0.026655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,4096,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,3584,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,3072,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,2560,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,2048,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,1536,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,1024,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,12288,0.04915200173854828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,10240,0.04297599941492081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,7168,0.03311999887228012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,8192,0.03711999952793121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,16384,0.06060799956321716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,65536,0.2096319943666458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,5120,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,4096,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,3072,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,3584,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,2560,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,2048,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,1536,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,256,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,1024,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,12288,0.04121600091457367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,10240,0.036031998693943024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,16384,0.04848000034689903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,7168,0.03097599931061268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,8192,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,65536,0.18704000115394592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,5120,0.026944000273942947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,4096,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,2048,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,2560,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,3072,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,3584,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,1536,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,1024,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,12288,0.036896001547575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,65536,0.14115199446678162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,7168,0.028704000636935234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,10240,0.033440001308918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,8192,0.030880000442266464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,16384,0.04201599955558777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,5120,0.027168000116944313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,2048,0.014495999552309513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,3584,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,3072,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,4096,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,2560,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,1536,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,1024,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,512,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,12288,0.037087999284267426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,8192,0.030559999868273735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,16384,0.039583999663591385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,7168,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,10240,0.03376000002026558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,65536,0.11766400188207626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,5120,0.024960000067949295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,4096,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,3072,0.01788800023496151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,3584,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,1536,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,1024,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,512,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,12288,0.03673600032925606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,8192,0.02739199995994568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,10240,0.03046399913728237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,7168,0.026784000918269157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,16384,0.03964800015091896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,65536,0.10790400207042694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,5120,0.02316799946129322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,3584,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,4096,0.021023999899625778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,2560,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,3072,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,12288,0.031199999153614044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,65536,0.09385599941015244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,8192,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,7168,0.02502400055527687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,10240,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,16384,0.035679999738931656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,5120,0.02287999913096428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,4096,0.02070399932563305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,3584,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,12288,0.0289280004799366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,8192,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,10240,0.0272000003606081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,7168,0.02457600086927414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,16384,0.03311999887228012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,65536,0.08767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,5120,0.023135999217629433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,3584,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,2048,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,2560,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,4096,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,128,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,12288,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,7168,0.02505600079894066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,8192,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,65536,0.0783040001988411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,10240,0.026016000658273697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,16384,0.03174399957060814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,5120,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,4096,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,3584,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,3072,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,1536,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,65536,0.06425599753856659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,12288,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,10240,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,8192,0.02332800067961216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,7168,0.022304000332951546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,16384,0.029600000008940697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,4096,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,5120,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,3072,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,3584,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,1536,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,65536,0.05260799825191498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,12288,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,8192,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,7168,0.02457600086927414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,10240,0.025151999667286873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,16384,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,4096,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,5120,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,3584,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,2560,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,2048,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,512,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,1536,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,65536,0.04556800052523613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,12288,0.033376000821590424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,8192,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,7168,0.026623999699950218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,16384,0.03542400151491165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,10240,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,4096,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,5120,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,3584,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,3072,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,2048,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,1024,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,65536,0.04169600084424019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,12288,0.034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,7168,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,10240,0.03244800120592117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,8192,0.026464000344276428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,16384,0.03545600175857544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,5120,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,4096,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,3584,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,3072,0.016767999157309532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,2560,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,256,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,65536,0.043296001851558685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,5120,0.12598399817943573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,4096,0.10390400141477585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,8192,0.19171200692653656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,10240,0.2399359941482544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,12288,0.2866879999637604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,7168,0.17043200135231018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,3584,0.09200000017881393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,2560,0.06825599819421768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,1024,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,1536,0.04572800174355507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,2048,0.05724800005555153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,3072,0.08115199953317642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,128,0.013279999606311321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,256,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,16384,0.3783999979496002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,512,0.02364799939095974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,12288,0.08179199695587158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,8192,0.05593600124120712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,5120,0.039712000638246536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,7168,0.05135999992489815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,16384,0.10252799838781357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,10240,0.07004799693822861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,4096,0.033695999532938004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,3584,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,2048,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,2560,0.024032000452280045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,1536,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,3072,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,1024,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,512,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,256,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,128,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,12288,0.06886400282382965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,8192,0.05004800111055374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,10240,0.058880001306533813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,7168,0.04374400153756142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,16384,0.08396799862384796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,5120,0.03292800113558769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,4096,0.03468799963593483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,3584,0.027583999559283257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,3072,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,2560,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,2048,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,1024,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,1536,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,65536,0.37721601128578186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,256,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,512,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,128,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,12288,0.0597120001912117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,8192,0.044096000492572784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,7168,0.03999999910593033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,10240,0.05119999870657921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,16384,0.07427199929952621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,65536,0.2903999984264374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,5120,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,4096,0.02598400041460991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,2560,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,3584,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,3072,0.02223999984562397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,2048,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,1536,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,512,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,256,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,1024,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,128,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,12288,0.04649600014090538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,10240,0.03996799886226654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,7168,0.0318400003015995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,8192,0.034912001341581345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,16384,0.05721599981188774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,65536,0.2659200131893158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,5120,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,4096,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,3072,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,2560,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,3584,0.01942400075495243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,1536,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,2048,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,1024,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,256,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,12288,0.04588799923658371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,7168,0.03145600110292435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,8192,0.03433600068092346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,16384,0.05696000158786774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,10240,0.04041599854826927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,65536,0.20268799364566803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,5120,0.029632000252604485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,3072,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,4096,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,3584,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,2560,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,2048,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,1536,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,1024,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,12288,0.03977600112557411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,10240,0.0363520011305809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,7168,0.02956799976527691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,8192,0.03215999901294708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,16384,0.047040000557899475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,65536,0.1781120002269745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,5120,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,4096,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,3584,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,3072,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,1536,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,12288,0.03526400029659271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,65536,0.13344000279903412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,16384,0.040383998304605484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,10240,0.03315199911594391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,7168,0.02659199945628643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,8192,0.029343999922275543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,5120,0.02364799939095974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,4096,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,3584,0.01881599985063076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,2048,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,1536,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,12288,0.034304000437259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,65536,0.11228799819946289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,8192,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,10240,0.0318400003015995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,7168,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,16384,0.03702399879693985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,4096,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,5120,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,3072,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,2560,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,1536,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,12288,0.033055998384952545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,10240,0.030527999624609947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,8192,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,7168,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,16384,0.035840000957250595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,65536,0.10707200318574905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,5120,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,4096,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,3584,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,2048,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,1024,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,12288,0.03126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,7168,0.02377600036561489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,8192,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,16384,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,10240,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,65536,0.08959999680519104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,5120,0.02070399932563305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,3584,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,4096,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,2560,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,128,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,12288,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,7168,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,65536,0.08182399719953537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,8192,0.024320000782608986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,10240,0.026176000013947487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,16384,0.031936001032590866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,5120,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,4096,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,2560,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,3584,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,12288,0.025472000241279602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,65536,0.07107199728488922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,8192,0.023296000435948372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,7168,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,10240,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,16384,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,5120,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,3584,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,4096,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,2560,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,3072,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,65536,0.060447998344898224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,12288,0.024960000067949295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,10240,0.024191999807953835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,16384,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,8192,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,7168,0.022752000018954277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,5120,0.022975999861955643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,4096,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,3584,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,3072,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,2048,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,65536,0.04825599864125252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,12288,0.030559999868273735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,8192,0.028351999819278717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,16384,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,10240,0.02924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,7168,0.02611199952661991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,5120,0.022655999287962914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,4096,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,3584,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,2048,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,3072,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,12288,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,65536,0.042367998510599136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,8192,0.027615999802947044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,10240,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,7168,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,16384,0.04137599840760231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,5120,0.02179200015962124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,4096,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,3584,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,3072,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,2560,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,2048,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,65536,0.04940799996256828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,12288,0.03359999880194664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,16384,0.04054399952292442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,8192,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,10240,0.029759999364614487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,7168,0.024159999564290047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,5120,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,4096,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,3584,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,3072,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,2560,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,2048,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,65536,0.05113599821925163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,5120,0.12348800152540207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,7168,0.16758400201797485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,4096,0.1013759970664978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,8192,0.19475199282169342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,12288,0.28409600257873535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,10240,0.2353920042514801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,3072,0.07859200239181519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,3584,0.09008000046014786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,2560,0.06860800087451935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,1536,0.0453759990632534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,2048,0.05628800019621849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,1024,0.033376000821590424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,512,0.02348800003528595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,16384,0.37049600481987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,128,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,256,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,12288,0.07993599772453308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,5120,0.03951999917626381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,7168,0.04982399940490723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,8192,0.05567999929189682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,10240,0.07078400254249573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,16384,0.10169599950313568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,3072,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,4096,0.03267199918627739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,3584,0.03171199932694435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,2048,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,2560,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,1536,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,512,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,1024,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,128,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,256,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,12288,0.06703999638557434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,8192,0.05132799968123436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,7168,0.041439998894929886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,10240,0.05363199859857559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,16384,0.08425600081682205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,4096,0.03126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,5120,0.03340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,3072,0.024064000695943832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,3584,0.025919999927282333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,2560,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,2048,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,1024,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,256,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,1536,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,512,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,65536,0.36873599886894226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,128,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,12288,0.056095998734235764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,8192,0.04182400181889534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,10240,0.05225599929690361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,7168,0.03807999938726425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,16384,0.06985600292682648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,4096,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,5120,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,65536,0.3112959861755371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,3584,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,2560,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,3072,0.02191999927163124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,2048,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,1536,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,1024,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,256,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,512,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,12288,0.04585599899291992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,8192,0.035071998834609985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,10240,0.03916800022125244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,7168,0.030079999938607216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,16384,0.05663999915122986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,4096,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,5120,0.024639999493956566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,3584,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,3072,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,65536,0.24463999271392822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,2560,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,2048,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,1536,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,12288,0.045152001082897186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,10240,0.038975998759269714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,7168,0.030079999938607216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,8192,0.033824000507593155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,16384,0.0551999993622303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,65536,0.1961279958486557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,5120,0.02879999950528145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,4096,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,3072,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,3584,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,2560,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,2048,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,1536,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,12288,0.0398080013692379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,7168,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,16384,0.04521600157022476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,10240,0.03593600168824196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,8192,0.030848000198602676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,65536,0.17801600694656372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,5120,0.024191999807953835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,3072,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,3584,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,4096,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,2048,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,12288,0.03488000109791756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,10240,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,65536,0.13318400084972382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,16384,0.0395519994199276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,7168,0.02598400041460991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,8192,0.02687999978661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,5120,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,4096,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,3584,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,3072,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,2048,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,2560,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,128,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,12288,0.03315199911594391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,8192,0.026976000517606735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,10240,0.02876799926161766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,16384,0.036896001547575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,7168,0.02505600079894066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,65536,0.10921599715948105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,5120,0.022784000262618065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,4096,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,3584,0.020864000543951988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,3072,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,1536,0.013279999606311321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,12288,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,8192,0.025760000571608543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,10240,0.028831999748945236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,65536,0.0990080013871193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,16384,0.03411199897527695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,7168,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,5120,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,4096,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,3584,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,3072,0.017503999173641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,2048,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,256,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,12288,0.0306560005992651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,65536,0.08627200126647949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,8192,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,10240,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,7168,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,16384,0.032255999743938446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,5120,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,4096,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,3584,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,3072,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,2048,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,12288,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,8192,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,10240,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,65536,0.0788159966468811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,7168,0.024191999807953835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,16384,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,5120,0.02364799939095974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,4096,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,3072,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,2560,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,2048,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,12288,0.02831999957561493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,10240,0.02691200003027916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,8192,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,65536,0.06739199906587601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,7168,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,16384,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,4096,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,5120,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,3584,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,3072,0.01744000054895878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,2560,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,1536,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,65536,0.05753599852323532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,12288,0.033055998384952545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,8192,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,10240,0.03215999901294708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,7168,0.025887999683618546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,16384,0.031808000057935715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,4096,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,5120,0.02364799939095974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,3584,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,3072,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,2048,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,2560,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,1024,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,12288,0.03622400015592575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,65536,0.053247999399900436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,8192,0.02687999978661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,10240,0.03033600002527237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,16384,0.04214400053024292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,7168,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,5120,0.022431999444961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,4096,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,2560,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,3584,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,2048,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,1024,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,12288,0.033663999289274216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,65536,0.05260799825191498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,7168,0.025760000571608543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,10240,0.03155200183391571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,8192,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,16384,0.04025600105524063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,3584,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,4096,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,3072,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,5120,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,2560,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,2048,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,512,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,1536,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,128,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,65536,0.07686399668455124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,12288,0.03209599852561951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,10240,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,8192,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,7168,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,16384,0.03824000060558319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,5120,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,4096,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,3584,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,3072,0.016767999157309532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,2560,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,1536,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,65536,0.07273600250482559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,4096,0.09907200187444687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,5120,0.12867200374603271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,7168,0.16937600076198578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,8192,0.19430400431156158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,10240,0.23894399404525757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,12288,0.2837119996547699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,3072,0.0791039988398552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,3584,0.08950400352478027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,1536,0.04556800052523613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,2560,0.06844799965620041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,2048,0.05641600117087364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,1024,0.03347200155258179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,128,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,256,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,512,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,16384,0.37465599179267883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,12288,0.08028800040483475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,8192,0.056832000613212585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,5120,0.03929600119590759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,7168,0.0498879998922348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,10240,0.06864000111818314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,16384,0.10115200281143188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,4096,0.032607998698949814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,3584,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,1536,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,2048,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,2560,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,3072,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,1024,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,256,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,512,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,128,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,12288,0.06310400366783142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,7168,0.04044799879193306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,8192,0.047775998711586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,10240,0.05321599915623665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,16384,0.07833600044250488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,5120,0.033663999289274216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,4096,0.028896000236272812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,3584,0.02800000086426735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,3072,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,2560,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,2048,0.017983999103307724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,1536,0.016448000445961952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,512,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,1024,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,256,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,65536,0.3911040127277374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,12288,0.056543998420238495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,7168,0.03734400123357773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,10240,0.053568001836538315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,8192,0.04131200164556503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,16384,0.0772479996085167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,5120,0.030719999223947525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,4096,0.02659199945628643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,3072,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,3584,0.024320000782608986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,2560,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,65536,0.27564799785614014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,2048,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,1536,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,1024,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,256,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,512,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,128,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,12288,0.04460800066590309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,10240,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,8192,0.035392001271247864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,7168,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,16384,0.05564799904823303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,5120,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,4096,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,3072,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,65536,0.26396799087524414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,3584,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,2560,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,2048,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,1536,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,512,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,1024,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,12288,0.04435199871659279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,8192,0.03315199911594391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,7168,0.029152000322937965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,16384,0.05532800033688545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,10240,0.038784001022577286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,65536,0.19657599925994873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,4096,0.02521600015461445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,5120,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,3072,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,3584,0.02271999977529049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,2560,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,1536,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,1024,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,2048,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,512,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,12288,0.0379519984126091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,10240,0.03215999901294708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,7168,0.028960000723600388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,16384,0.04575999826192856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,8192,0.030912000685930252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,5120,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,65536,0.18137599527835846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,4096,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,3072,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,3584,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,2560,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,1536,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,2048,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,512,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,12288,0.034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,10240,0.031199999153614044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,16384,0.03923200070858002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,7168,0.023840000852942467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,65536,0.1356479972600937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,8192,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,5120,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,2560,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,4096,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,2048,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,3072,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,3584,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,1536,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,1024,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,12288,0.0318400003015995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,16384,0.03667199984192848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,65536,0.11164800077676773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,8192,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,7168,0.023072000592947006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,10240,0.028543999418616295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,5120,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,3584,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,3072,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,4096,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,2048,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,1024,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,128,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,12288,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,8192,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,10240,0.023264000192284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,65536,0.09587199985980988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,16384,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,7168,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,5120,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,4096,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,3584,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,2560,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,3072,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,2048,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,1536,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,512,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,1024,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,12288,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,8192,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,10240,0.022655999287962914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,7168,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,65536,0.08694399893283844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,16384,0.03299200162291527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,5120,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,4096,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,3584,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,3072,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,2048,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,2560,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,1536,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,512,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,1024,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,12288,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,8192,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,10240,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,7168,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,65536,0.07887999713420868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,16384,0.02470399998128414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,5120,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,3072,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,2048,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,3584,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,4096,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,2560,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,1536,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,512,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,1024,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,256,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,128,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,12288,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,8192,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,65536,0.0663679987192154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,10240,0.01759999990463257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,7168,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,16384,0.023903999477624893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,5120,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,3072,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,2048,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,4096,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,2560,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,3584,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,1536,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,512,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,256,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,1024,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,12288,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,65536,0.05644800141453743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,7168,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,8192,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,10240,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,16384,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,5120,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,4096,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,3584,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,2048,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,3072,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,2560,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,1536,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,256,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,512,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,128,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,1024,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,65536,0.05023999884724617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,12288,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,8192,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,7168,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,16384,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,10240,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,5120,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,4096,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,3584,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,3072,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,2560,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,2048,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,1024,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,1536,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,512,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,128,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,256,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,65536,0.05596800148487091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,12288,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,8192,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,10240,0.013535999692976475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,16384,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,7168,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,5120,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,4096,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,3584,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,3072,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,2560,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,1536,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,2048,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,1024,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,512,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,128,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,256,0.008128000423312187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,65536,0.057151999324560165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,12288,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,16384,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,7168,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,10240,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,8192,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,5120,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,4096,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,3072,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,2560,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,3584,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,2048,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,1536,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,1024,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,256,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,512,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,128,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,65536,0.053119998425245285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,5120,0.12332800030708313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,4096,0.0968639999628067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,7168,0.16310399770736694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,8192,0.17936000227928162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,10240,0.2252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,12288,0.2686080038547516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,3072,0.07872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,2560,0.06896000355482101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,3584,0.08675199747085571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,2048,0.05580800026655197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,1536,0.043935999274253845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,16384,0.3712320029735565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,1024,0.033440001308918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,512,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,256,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,128,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,12288,0.07843200117349625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,8192,0.05488000065088272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,7168,0.05161599814891815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,5120,0.04022400081157684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,10240,0.06480000168085098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,16384,0.09779199957847595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,4096,0.03267199918627739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,3072,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,2560,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,3584,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,2048,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,1536,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,1024,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,512,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,256,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,128,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,12288,0.06201599910855293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,7168,0.03993599861860275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,10240,0.050912000238895416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,8192,0.04275200143456459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,16384,0.07603199779987335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,5120,0.030688000842928886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,4096,0.026944000273942947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,3584,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,3072,0.023840000852942467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,2560,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,2048,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,1024,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,1536,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,512,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,65536,0.37276801466941833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,256,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,12288,0.05615999922156334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,16384,0.07859200239181519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,8192,0.037856001406908035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,10240,0.04713600128889084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,7168,0.03593600168824196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,5120,0.027648000046610832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,65536,0.28012800216674805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,4096,0.02377600036561489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,3072,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,2560,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,3584,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,2048,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,512,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,1024,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,1536,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,256,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,12288,0.04460800066590309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,8192,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,10240,0.03731200098991394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,7168,0.028575999662280083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,16384,0.05276799947023392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,5120,0.02396799996495247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,4096,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,65536,0.24643200635910034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,3584,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,3072,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,2560,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,2048,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,1536,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,1024,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,12288,0.04022400081157684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,7168,0.026623999699950218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,10240,0.03532800078392029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,8192,0.028991999104619026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,16384,0.04982399940490723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,5120,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,3584,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,4096,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,65536,0.18729600310325623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,2560,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,512,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,1536,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,128,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,12288,0.03465599939227104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,8192,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,16384,0.041120000183582306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,10240,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,7168,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,65536,0.17615999281406403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,5120,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,3584,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,2560,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,4096,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,3072,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,2048,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,1536,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,1024,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,512,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,12288,0.02611199952661991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,16384,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,7168,0.01759999990463257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,10240,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,8192,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,65536,0.13443200290203094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,5120,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,4096,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,3072,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,2560,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,2048,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,3584,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,1536,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,1024,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,512,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,128,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,256,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,12288,0.023296000435948372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,65536,0.0981760025024414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,8192,0.01788800023496151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,10240,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,16384,0.028575999662280083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,7168,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,5120,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,3584,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,4096,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,2560,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,3072,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,2048,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,1536,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,1024,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,256,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,128,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,512,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,12288,0.022336000576615334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,10240,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,8192,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,65536,0.09305600076913834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,16384,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,7168,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,5120,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,4096,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,3584,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,2560,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,3072,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,2048,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,1536,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,1024,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,256,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,512,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,128,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,12288,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,8192,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,10240,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,7168,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,16384,0.02332800067961216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,65536,0.08348800241947174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,5120,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,2560,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,4096,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,3072,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,2048,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,3584,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,1536,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,1024,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,256,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,512,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,128,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,12288,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,65536,0.07654400169849396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,8192,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,7168,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,10240,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,16384,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,5120,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,4096,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,2048,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,3584,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,3072,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,2560,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,1536,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,512,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,256,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,1024,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,128,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,65536,0.06595200300216675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,12288,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,16384,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,10240,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,8192,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,7168,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,5120,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,4096,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,3584,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,3072,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,2560,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,1024,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,1536,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,2048,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,512,0.007007999811321497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,256,0.007040000054985285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,128,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,12288,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,65536,0.058079998940229416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,16384,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,10240,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,8192,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,7168,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,5120,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,4096,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,3072,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,3584,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,2560,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,2048,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,256,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,512,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,128,0.0071680000983178616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,1024,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,1536,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,65536,0.05008000135421753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,12288,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,7168,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,10240,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,8192,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,16384,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,4096,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,5120,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,3584,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,2560,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,3072,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,2048,0.007615999784320593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,1536,0.007071999832987785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,1024,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,128,0.006976000033318996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,256,0.00687999976798892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,512,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,12288,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,65536,0.044256001710891724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,16384,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,7168,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,8192,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,10240,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,5120,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,4096,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,3584,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,1536,0.007424000184983015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,2048,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,2560,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,3072,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,1024,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,512,0.006976000033318996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,256,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,128,0.006976000033318996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,65536,0.04278400167822838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,12288,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,8192,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,10240,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,16384,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,7168,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,5120,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,4096,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,3072,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,3584,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,2560,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,2048,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,1536,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,512,0.00687999976798892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,256,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,1024,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,128,0.006943999789655209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,65536,0.04278400167822838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,5120,0.11423999816179276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,4096,0.09196799993515015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,7168,0.15638400614261627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,8192,0.17558400332927704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,10240,0.21721599996089935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,12288,0.2610880136489868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,2048,0.05196800082921982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,3072,0.0719040036201477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,3584,0.08256000280380249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,2560,0.06784000247716904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,1024,0.03251200169324875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,1536,0.042688000947237015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,16384,0.34329599142074585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,512,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,256,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,128,0.013279999606311321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,12288,0.07158400118350983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,8192,0.049984000623226166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,5120,0.034752000123262405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,7168,0.04508800059556961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,10240,0.0605119988322258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,16384,0.09296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,4096,0.029184000566601753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,3584,0.027583999559283257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,3072,0.024447999894618988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,2560,0.023711999878287315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,2048,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,1536,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,512,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,1024,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,128,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,256,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,12288,0.05571199953556061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,10240,0.04739199951291084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,7168,0.03532800078392029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,8192,0.03993599861860275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,16384,0.0708480030298233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,5120,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,4096,0.02377600036561489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,2560,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,3072,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,3584,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,2048,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,1536,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,1024,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,256,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,128,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,65536,0.34595200419425964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,12288,0.04867200180888176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,8192,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,10240,0.04108799993991852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,7168,0.031808000057935715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,16384,0.06201599910855293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,5120,0.024159999564290047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,4096,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,3584,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,3072,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,2560,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,65536,0.2609280049800873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,128,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,256,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,12288,0.04137599840760231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,7168,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,8192,0.0306560005992651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,10240,0.03558399900794029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,16384,0.05385600030422211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,5120,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,4096,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,65536,0.22099199891090393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,3072,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,2560,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,3584,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,2048,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,1536,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,1024,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,12288,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,7168,0.02582399919629097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,8192,0.028575999662280083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,10240,0.033952001482248306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,16384,0.04960000142455101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,65536,0.1887039989233017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,5120,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,3072,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,3584,0.01651199907064438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,2560,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,4096,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,2048,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,1536,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,128,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,512,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,1024,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,12288,0.027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,16384,0.03468799963593483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,7168,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,8192,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,10240,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,65536,0.17366400361061096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,5120,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,4096,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,3584,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,3072,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,2560,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,2048,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,1536,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,1024,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,512,0.007648000027984381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,256,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,128,0.0074880002066493034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,12288,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,65536,0.11430399864912033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,10240,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,16384,0.03097599931061268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,8192,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,7168,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,5120,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,4096,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,3584,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,3072,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,2560,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,1536,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,2048,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,1024,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,256,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,512,0.007552000228315592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,128,0.007648000027984381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,12288,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,10240,0.01942400075495243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,8192,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,16384,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,65536,0.09510400146245956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,7168,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,5120,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,4096,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,3584,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,3072,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,2560,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,1536,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,2048,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,1024,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,512,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,128,0.007615999784320593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,256,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,12288,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,65536,0.08470399677753448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,8192,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,7168,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,10240,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,16384,0.024320000782608986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,5120,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,4096,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,3584,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,2048,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,3072,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,2560,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,1536,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,1024,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,512,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,128,0.0071680000983178616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,256,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,65536,0.07436800003051758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,12288,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,16384,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,10240,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,8192,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,7168,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,4096,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,5120,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,3584,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,3072,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,2048,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,1536,0.008128000423312187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,2560,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,1024,0.007679999805986881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,512,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,256,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,128,0.007040000054985285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,12288,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,65536,0.06646399945020676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,16384,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,10240,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,8192,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,5120,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,7168,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,4096,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,2560,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,3072,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,3584,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,2048,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,1024,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,1536,0.007679999805986881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,256,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,512,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,128,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,12288,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,65536,0.05926400050520897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,8192,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,7168,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,10240,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,16384,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,5120,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,4096,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,2560,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,3584,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,3072,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,2048,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,1024,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,1536,0.007552000228315592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,512,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,256,0.007071999832987785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,128,0.006912000011652708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,12288,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,65536,0.05270399898290634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,10240,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,16384,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,8192,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,7168,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,5120,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,4096,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,2560,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,3584,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,3072,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,2048,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,1024,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,1536,0.0071680000983178616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,128,0.007615999784320593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,256,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,512,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,12288,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,65536,0.0480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,8192,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,16384,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,7168,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,10240,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,4096,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,5120,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,2048,0.007040000054985285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,3584,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,2560,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,3072,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,1536,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,1024,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,256,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,512,0.00687999976798892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,128,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,65536,0.044576000422239304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,12288,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,8192,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,7168,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,10240,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,16384,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,4096,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,5120,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,3584,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,3072,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,2560,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,2048,0.007040000054985285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,1536,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,1024,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,512,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,256,0.006752000190317631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,128,0.006783999968320131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,65536,0.04320000112056732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,12288,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,10240,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,7168,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,8192,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,5120,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,16384,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,4096,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,2560,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,3072,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,3584,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,2048,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,1536,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,1024,0.006943999789655209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,128,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,256,0.007007999811321497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,512,0.006943999789655209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,65536,0.04249599948525429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,5120,0.11257600039243698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,4096,0.09094399958848953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,7168,0.15564799308776855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,8192,0.17478400468826294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,12288,0.2627840042114258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,10240,0.21769599616527557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,2560,0.0650240033864975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,3584,0.08051200211048126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,3072,0.07072000205516815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,2048,0.048576001077890396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,1536,0.039264000952243805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,256,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,1024,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,512,0.022975999861955643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,16384,0.3405439853668213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,128,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,7168,0.044863998889923096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,5120,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,8192,0.05190400034189224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,12288,0.07171200215816498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,10240,0.05984000116586685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,16384,0.0926399976015091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,4096,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,3584,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,2560,0.023391999304294586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,3072,0.02332800067961216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,2048,0.017503999173641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,1024,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,512,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,1536,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,128,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,256,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,12288,0.05913599953055382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,8192,0.042047999799251556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,7168,0.03753599897027016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,10240,0.049056001007556915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,16384,0.07542400062084198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,4096,0.023679999634623528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,3584,0.02179200015962124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,5120,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,3072,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,2560,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,2048,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,1536,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,1024,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,256,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,512,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,65536,0.3491840064525604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,128,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,12288,0.05484800040721893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,8192,0.03788800165057182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,7168,0.036159999668598175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,10240,0.04713600128889084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,16384,0.07209599763154984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,5120,0.02723200060427189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,4096,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,65536,0.27929601073265076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,3584,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,3072,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,2560,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,1536,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,512,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,256,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,1024,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,128,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,12288,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,8192,0.028672000393271446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,7168,0.025919999927282333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,10240,0.03385600075125694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,16384,0.05180799961090088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,5120,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,4096,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,3584,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,2560,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,65536,0.24435199797153473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,3072,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,2048,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,1024,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,256,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,512,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,128,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,1536,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,12288,0.03683200106024742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,8192,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,10240,0.030559999868273735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,7168,0.024000000208616257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,16384,0.04553600028157234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,65536,0.18188799917697906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,5120,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,4096,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,3584,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,3072,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,2560,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,2048,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,1536,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,1024,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,512,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,128,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,256,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,12288,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,10240,0.024159999564290047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,8192,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,7168,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,16384,0.035392001271247864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,65536,0.15964800119400024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,5120,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,3584,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,4096,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,3072,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,2560,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,2048,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,1024,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,512,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,1536,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,128,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,256,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,12288,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,10240,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,16384,0.03152000159025192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,8192,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,7168,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,65536,0.11923199892044067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,5120,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,4096,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,3584,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,3072,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,2560,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,1536,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,2048,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,512,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,1024,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,256,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,128,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,12288,0.02377600036561489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,8192,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,65536,0.10265599936246872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,16384,0.029823999851942062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,10240,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,7168,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,5120,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,4096,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,3072,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,3584,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,2560,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,2048,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,1536,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,256,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,1024,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,512,0.008128000423312187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,128,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,12288,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,16384,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,65536,0.09571199864149094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,8192,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,7168,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,10240,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,5120,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,3584,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,3072,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,2048,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,4096,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,2560,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,1536,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,1024,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,512,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,128,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,256,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,12288,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,8192,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,65536,0.08748800307512283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,7168,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,10240,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,16384,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,4096,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,5120,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,3584,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,2048,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,3072,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,2560,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,1024,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,256,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,128,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,512,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,1536,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,12288,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,65536,0.07558400183916092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,7168,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,8192,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,10240,0.01788800023496151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,16384,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,4096,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,3584,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,5120,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,2560,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,2048,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,3072,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,1024,0.007679999805986881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,512,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,1536,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,128,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,256,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,12288,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,65536,0.06377600133419037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,8192,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,7168,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,10240,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,16384,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,5120,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,4096,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,3072,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,3584,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,2048,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,2560,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,1024,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,1536,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,512,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,256,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,128,0.007712000049650669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,12288,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,65536,0.057760000228881836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,8192,0.014495999552309513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,7168,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,10240,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,16384,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,5120,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,4096,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,2560,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,3072,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,3584,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,2048,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,1536,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,256,0.007648000027984381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,128,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,512,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,1024,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,65536,0.04960000142455101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,12288,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,8192,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,7168,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,10240,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,16384,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,4096,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,5120,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,3584,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,2560,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,3072,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,2048,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,1536,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,1024,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,512,0.007648000027984381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,256,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,128,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,12288,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,65536,0.0716480016708374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,10240,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,16384,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,8192,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,7168,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,4096,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,5120,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,3584,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,2560,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,3072,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,2048,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,1024,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,1536,0.007552000228315592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,512,0.007712000049650669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,128,0.0074880002066493034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,256,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,65536,0.07145600020885468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,12288,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,8192,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,7168,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,10240,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,16384,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,5120,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,4096,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,3584,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,2560,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,3072,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,2048,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,1536,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,1024,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,512,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,256,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,128,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,65536,0.07110399752855301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,4096,5.157919883728027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,5120,6.444159984588623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,7168,8.535967826843262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,3584,4.532735824584961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,8192,9.975199699401855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,3072,3.91759991645813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,1024,1.49344003200531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,2560,3.3286080360412598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,1536,2.0918400287628174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,2048,2.689055919647217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,512,0.915008008480072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,128,0.42668798565864563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,256,0.604960024356842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,10240,12.599360466003418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,12288,3.6092801094055176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,7168,2.1540799140930176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,12288,14.690655708312988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,8192,2.4443519115448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,10240,3.024127960205078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,5120,1.5721919536590576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,4096,1.2760319709777832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,3584,1.1340160369873047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,1536,0.5344640016555786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,2560,0.8362240195274353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,3072,0.9913920164108276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,2048,0.6857920289039612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,256,0.15667200088500977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,128,0.11110399663448334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,512,0.24409599602222443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,1024,0.38841599225997925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,16384,4.769279956817627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,16384,20.984607696533203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,8192,1.8423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,10240,2.2778561115264893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,12288,2.7152318954467773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,7168,1.6235840320587158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,4096,0.9667519927024841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,5120,1.186560034751892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,3584,0.8603519797325134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,16384,3.5854079723358154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,2048,0.5210880041122437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,1024,0.2986240088939667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,2560,0.6362879872322083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,1536,0.41068801283836365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,3072,0.7485120296478271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,256,0.11990399658679962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,128,0.08774399757385254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,512,0.18934400379657745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,8192,1.5420160293579102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,10240,1.90447998046875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,12288,2.2721920013427734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,16384,3.000864028930664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,7168,1.35862398147583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,5120,0.9976000189781189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,4096,0.8097599744796753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,3072,0.6281279921531677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,2560,0.5327360033988953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,1536,0.3442240059375763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,2048,0.4374080002307892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,3584,0.7205439805984497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,256,0.09929600358009338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,128,0.07302399724721909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,512,0.160288006067276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,1024,0.25516799092292786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,10240,1.5606720447540283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,12288,1.8623039722442627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,16384,2.45849609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,7168,1.1127680540084839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,8192,1.2623679637908936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,5120,0.8148800134658813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,3584,0.5908160209655762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,4096,0.661408007144928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,3072,0.5147519707679749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,1536,0.27769601345062256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,2048,0.36208000779151917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,2560,0.4366079866886139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,1024,0.20419199764728546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,256,0.08588799834251404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,512,0.12908799946308136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,128,0.058880001306533813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,65536,21.719263076782227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,12288,2.2238080501556396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,16384,3.5719680786132812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,10240,1.367967963218689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,8192,1.1075199842453003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,7168,0.9778879880905151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,65536,16.286624908447266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,3584,0.5195519924163818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,5120,0.7145599722862244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,4096,0.582431972026825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,3072,0.4533439874649048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,2560,0.38547199964523315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,1536,0.24563199281692505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,2048,0.3187200129032135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,512,0.1151999980211258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,256,0.07430399954319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,1024,0.18143999576568604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,128,0.0549440011382103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,65536,13.942432403564453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,12288,1.1770559549331665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,10240,0.9803839921951294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,8192,0.7963520288467407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,16384,1.5430400371551514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,65536,11.31174373626709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,5120,0.5142719745635986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,4096,0.41865599155426025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,7168,0.7004160284996033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,2048,0.23126399517059326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,2560,0.2775680124759674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,1536,0.17878399789333344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,3584,0.37462401390075684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,3072,0.3264319896697998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,256,0.05657599866390228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,128,0.041600000113248825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,1024,0.13283200562000275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,512,0.08560000360012054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,8192,0.9201279878616333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,12288,1.3710720539093018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,10240,0.7863680124282837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,16384,1.6404800415039062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,7168,0.7591999769210815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,5120,0.4115839898586273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,3584,0.3198719918727875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,3072,0.2624000012874603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,4096,0.3375360071659088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,2048,0.1857600063085556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,1024,0.10796800255775452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,2560,0.22390399873256683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,1536,0.14508800208568573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,512,0.07427199929952621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,128,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,256,0.04793599992990494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,10240,0.6877440214157104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,12288,0.81740802526474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,65536,9.9999361038208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,65536,7.074624061584473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,16384,1.0800319910049438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,8192,0.555296003818512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,7168,0.48972800374031067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,5120,0.35916799306869507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,4096,0.2932800054550171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,3584,0.2609280049800873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,1024,0.09651199728250504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,2048,0.16332800686359406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,1536,0.1308480054140091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,2560,0.19491200149059296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,3072,0.22784000635147095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,256,0.04217600077390671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,512,0.06489600241184235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,128,0.03270399942994118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,8192,0.4776960015296936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,10240,0.5903679728507996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,65536,5.555903911590576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,12288,0.7023040056228638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,16384,0.9293439984321594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,7168,0.42131200432777405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,3584,0.2250880002975464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,4096,0.25283199548721313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,5120,0.3091840147972107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,3072,0.19660800695419312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,512,0.0560000017285347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,1024,0.0835840031504631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,1536,0.11164800077676773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,2048,0.14035199582576752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,2560,0.16835199296474457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,128,0.028736000880599022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,256,0.03776000067591667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,8192,0.3999040126800537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,10240,0.49372801184654236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,12288,0.5869759917259216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,16384,0.774944007396698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,5120,0.2596159875392914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,7168,0.35334399342536926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,4096,0.212351992726326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,2048,0.11785600334405899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,2560,0.14188799262046814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,3072,0.16595199704170227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,3584,0.18979200720787048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,65536,4.968128204345703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,256,0.033504001796245575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,512,0.04809600114822388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,1024,0.07110399752855301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,1536,0.09471999853849411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,128,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,8192,0.32156801223754883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,10240,0.3957119882106781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,12288,0.4719359874725342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,65536,4.363264083862305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,7168,0.2834239900112152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,16384,0.6259199976921082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,5120,0.2091519981622696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,4096,0.17180800437927246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,3584,0.152319997549057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,2048,0.09628800302743912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,3072,0.1335040032863617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,1536,0.07708799839019775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,2560,0.1151999980211258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,1024,0.058079998940229416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,128,0.023584000766277313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,256,0.028736000880599022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,512,0.039103999733924866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,8192,0.24320000410079956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,10240,0.2996799945831299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,12288,0.35686400532722473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,16384,0.46911999583244324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,5120,0.15929600596427917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,7168,0.21561600267887115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,4096,0.13158400356769562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,2048,0.07289600372314453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,3584,0.11584000289440155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,2560,0.08787199854850769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,3072,0.10182400047779083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,256,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,512,0.029791999608278275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,1024,0.04399999976158142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,1536,0.058559998869895935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,65536,3.692863941192627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,128,0.018079999834299088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,8192,0.16595199704170227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,10240,0.2030400037765503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,12288,0.2417919933795929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,16384,0.31673601269721985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,5120,0.1093439981341362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,4096,0.09036800265312195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,7168,0.14668799936771393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,2048,0.051711998879909515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,2560,0.061535999178886414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,3584,0.08102399855852127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,3072,0.07088000327348709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,65536,2.875296115875244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,256,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,512,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,1024,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,1536,0.04137599840760231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,128,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,8192,0.08816000074148178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,10240,0.14112000167369843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,12288,0.16550399363040924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,16384,0.16380800306797028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,5120,0.07612799853086472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,4096,0.06345599889755249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,7168,0.07913599908351898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,65536,2.191999912261963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,2560,0.035711999982595444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,3072,0.050783999264240265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,3584,0.045823998749256134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,2048,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,256,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,512,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,1536,0.0261439997702837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,1024,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,128,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,8192,0.061344001442193985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,10240,0.09219200164079666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,65536,1.4120639562606812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,12288,0.08585599809885025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,3584,0.03683200106024742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,4096,0.03759999945759773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,5120,0.04835199937224388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,16384,0.10995200276374817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,7168,0.05532800033688545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,1536,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,1024,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,2048,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,2560,0.02579200081527233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,3072,0.03049599938094616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,256,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,128,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,512,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,8192,0.0575999990105629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,10240,0.06937599927186966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,12288,0.081216000020504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,16384,0.10268799960613251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,5120,0.04124800115823746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,4096,0.03440000116825104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,7168,0.05132799968123436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,65536,0.715936005115509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,3584,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,2048,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,3072,0.027648000046610832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,2560,0.02271999977529049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,1536,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,512,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,1024,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,128,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,256,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,65536,0.47231999039649963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,7168,4.472544193267822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,65536,0.3561600148677826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,8192,5.0854082107543945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,5120,3.2489919662475586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,10240,5.9897918701171875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,3584,2.2714240550994873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,4096,2.5981760025024414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,3072,1.9734400510787964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,2560,1.664736032485962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,12288,7.763807773590088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,256,0.29071998596191406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,512,0.466048002243042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,2048,1.3509440422058105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,128,0.22752000391483307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,1024,0.7585920095443726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,1536,1.0476160049438477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,8192,1.2539199590682983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,10240,1.552351951599121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,16384,2.449536085128784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,5120,0.8010560274124146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,3584,0.5780479907989502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,3072,0.5031999945640564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,7168,1.1012799739837646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,4096,0.6527360081672668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,2048,0.3537920117378235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,2560,0.4275839924812317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,512,0.12947200238704681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,1024,0.20345599949359894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,256,0.0751039981842041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,1536,0.27900800108909607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,12288,1.8659199476242065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,128,0.05923200026154518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,16384,9.477824211120605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,7168,0.83024001121521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,8192,0.9411519765853882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,10240,1.1661759614944458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,12288,1.3908480405807495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,4096,0.49350398778915405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,5120,0.6044480204582214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,3584,0.4373440146446228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,3072,0.3807680010795593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,16384,1.8417919874191284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,2048,0.26822400093078613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,2560,0.324288010597229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,1536,0.2134079933166504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,512,0.1005759984254837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,1024,0.15670399367809296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,256,0.05926400050520897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,128,0.04854400083422661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,12288,1.1633599996566772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,8192,0.7895680069923401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,7168,0.6935999989509583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,10240,0.9735360145568848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,4096,0.41465601325035095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,3584,0.3665919899940491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,5120,0.5073599815368652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,3072,0.32067200541496277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,16384,1.5373120307922363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,2048,0.22617599368095398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,1536,0.17951999604701996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,2560,0.2720640003681183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,128,0.04556800052523613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,1024,0.13276800513267517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,256,0.0514880008995533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,512,0.08591999858617783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,12288,0.9303680062294006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,10240,0.7875199913978577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,8192,0.6312959790229797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,5120,0.4066559970378876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,16384,1.2317440509796143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,7168,0.5610880255699158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,4096,0.33158400654792786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,3584,0.2956480085849762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,2560,0.2202879935503006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,3072,0.2584959864616394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,2048,0.18211199343204498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,1536,0.14483200013637543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,512,0.07027199864387512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,1024,0.10831999778747559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,256,0.0427200011909008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,128,0.03283200040459633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,65536,5.299136161804199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,12288,0.8165760040283203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,65536,6.8766398429870605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,16384,1.078112006187439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,10240,0.6845759749412537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,8192,0.5549119710922241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,5120,0.357695996761322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,65536,7.56112003326416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,65536,11.232319831848145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,7168,0.48899200558662415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,4096,0.29238399863243103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,3584,0.25996801257133484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,3072,0.22627200186252594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,2560,0.19340799748897552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,1536,0.1287360042333603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,2048,0.16131199896335602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,1024,0.0958079993724823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,512,0.06883200258016586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,256,0.038816001266241074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,128,0.03001599945127964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,7168,0.3521920144557953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,10240,0.49241599440574646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,8192,0.39926400780677795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,5120,0.258432000875473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,16384,0.7732800245285034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,12288,0.5873919725418091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,4096,0.21216000616550446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,3584,0.1881600022315979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,1536,0.1111999973654747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,2048,0.11779200285673141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,2560,0.14115199446678162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,3072,0.16470399498939514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,1024,0.07152000069618225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,128,0.022975999861955643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,256,0.03046399913728237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,512,0.048128001391887665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,12288,0.4708479940891266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,8192,0.3210560083389282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,10240,0.39635199308395386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,5120,0.21007999777793884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,16384,0.6210880279541016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,7168,0.28646400570869446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,4096,0.17155200242996216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,3584,0.15302400290966034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,2560,0.11488000303506851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,2048,0.09603200107812881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,3072,0.13331200182437897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,1536,0.07868800312280655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,512,0.040991999208927155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,1024,0.05852799862623215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,256,0.02579200081527233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,128,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,65536,4.778495788574219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,10240,0.35337600111961365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,12288,0.41571199893951416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,16384,0.5443840026855469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,8192,0.28675198554992676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,5120,0.18982400000095367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,7168,0.2558079957962036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,4096,0.15881599485874176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,3584,0.143327996134758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,3072,0.12646399438381195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,2560,0.11196800321340561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,2048,0.09417600184679031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,1024,0.058880001306533813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,1536,0.07702399790287018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,512,0.037408001720905304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,128,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,256,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,65536,3.444000005722046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,65536,2.419424057006836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,10240,0.29996800422668457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,12288,0.35600000619888306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,8192,0.2654719948768616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,4096,0.13052800297737122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,5120,0.1589760035276413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,16384,0.46911999583244324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,3584,0.12464000284671783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,7168,0.21571199595928192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,2048,0.0751039981842041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,1536,0.06111999973654747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,1024,0.057920001447200775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,2560,0.08902399986982346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,3072,0.10246399790048599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,256,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,128,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,512,0.03296000137925148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,8192,0.21174399554729462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,10240,0.25862398743629456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,12288,0.3071039915084839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,65536,2.3669118881225586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,16384,0.4012799859046936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,4096,0.11974400281906128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,5120,0.14246399700641632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,7168,0.18863999843597412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,3072,0.09708800166845322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,3584,0.10864000022411346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,1536,0.06047999858856201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,2048,0.07360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,2560,0.08611200004816055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,512,0.02908799983561039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,256,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,128,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,1024,0.045951999723911285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,8192,0.16518400609493256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,12288,0.25884801149368286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,10240,0.2032639980316162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,5120,0.11007999628782272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,16384,0.315775990486145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,4096,0.0907839983701706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,7168,0.14633600413799286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,65536,1.8180480003356934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,2048,0.052191998809576035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,3072,0.07478400319814682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,2560,0.06233600154519081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,3584,0.08204799890518188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,1536,0.04374400153756142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,512,0.02521600015461445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,256,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,128,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,1024,0.03379200026392937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,8192,0.13439999520778656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,10240,0.16182400286197662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,12288,0.1884160041809082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,7168,0.12067200243473053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,16384,0.24323199689388275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,65536,1.5831680297851562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,4096,0.07887999713420868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,5120,0.09321600198745728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,3584,0.07289600372314453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,3072,0.06489600241184235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,2560,0.057792000472545624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,1536,0.041152000427246094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,2048,0.05004800111055374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,1024,0.03209599852561951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,256,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,128,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,512,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,65536,1.400607943534851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,8192,0.08694399893283844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,7168,0.07788799703121185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,10240,0.10652799904346466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,12288,0.12422399967908859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,5120,0.05907199904322624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,4096,0.04975999891757965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,16384,0.1621440052986145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,3584,0.04492799937725067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,3072,0.040031999349594116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,2560,0.058111999183893204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,512,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,1024,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,2048,0.0306560005992651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,1536,0.025760000571608543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,256,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,128,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,8192,0.07366400212049484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,12288,0.07315199822187424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,10240,0.06339199841022491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,16384,0.09228800237178802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,5120,0.04966399818658829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,4096,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,7168,0.0650240033864975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,65536,0.9847679734230042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,3584,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,2560,0.022784000262618065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,3072,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,2048,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,1536,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,512,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,1024,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,128,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,256,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,8192,0.04022400081157684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,10240,0.07091200351715088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,65536,0.6115840077400208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,12288,0.08416000008583069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,3584,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,16384,0.06703999638557434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,4096,0.025599999353289604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,5120,0.02924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,7168,0.036479998379945755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,2048,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,2560,0.01942400075495243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,3072,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,1536,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,1024,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,128,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,256,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,512,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,65536,0.45395201444625854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,8192,0.03683200106024742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,10240,0.04416000097990036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,12288,0.049247998744249344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,16384,0.06281600147485733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,7168,0.03363199904561043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,5120,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,3584,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,4096,0.022495999932289124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,2560,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,2048,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,3072,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,1536,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,256,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,512,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,128,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,8192,2.5060479640960693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,65536,0.21135999262332916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,16384,4.893152236938477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,4096,1.2622079849243164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,3584,1.1376960277557373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,3072,0.9890559911727905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,65536,0.20918400585651398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,5120,1.5539840459823608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,7168,2.219104051589966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,512,0.24515199661254883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,256,0.12880000472068787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,1024,0.389055997133255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,2048,0.6857920289039612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,1536,0.5334720015525818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,128,0.11420799791812897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,2560,0.8324800133705139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,10240,3.1541759967803955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,10240,0.7821120023727417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,7168,0.5563520193099976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,12288,3.862816095352173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,8192,0.6337599754333496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,12288,0.9303680062294006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,5120,0.4062719941139221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,4096,0.33161601424217224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,3584,0.29577600955963135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,2048,0.18201600015163422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,3072,0.25734400749206543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,1024,0.10787200182676315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,1536,0.1451839953660965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,2560,0.21904000639915466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,512,0.07414399832487106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,128,0.034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,256,0.03923200070858002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,16384,1.2306239604949951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,12288,0.7009599804878235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,10240,0.5889599919319153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,5120,0.3075839877128601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,8192,0.47654399275779724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,4096,0.2523519992828369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,7168,0.4219520092010498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,65536,4.98092794418335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,3072,0.19599999487400055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,3584,0.22377599775791168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,2560,0.16806399822235107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,1024,0.084927998483181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,512,0.05286400020122528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,16384,0.9264640212059021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,1536,0.11276800185441971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,2048,0.14003199338912964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,256,0.03283200040459633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,128,0.028896000236272812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,7168,0.3524160087108612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,16384,0.7728319764137268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,8192,0.39900800585746765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,5120,0.25859200954437256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,10240,0.4971520006656647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,12288,0.5864959955215454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,65536,3.7177278995513916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,4096,0.21129600703716278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,3584,0.18873600661754608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,3072,0.16495999693870544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,2048,0.118367999792099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,1024,0.08355200290679932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,1536,0.09494400024414062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,2560,0.14188799262046814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,512,0.045504000037908554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,256,0.02864000014960766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,128,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,8192,0.320576012134552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,10240,0.3957439959049225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,7168,0.2852799892425537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,12288,0.4742079973220825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,4096,0.17164799571037292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,5120,0.20902399718761444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,3584,0.15292799472808838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,3072,0.133760005235672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,16384,0.6208000183105469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,2560,0.115167997777462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,2048,0.09616000205278397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,1024,0.05878400057554245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,1536,0.07769600301980972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,65536,3.022144079208374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,512,0.03862399980425835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,256,0.02396799996495247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,128,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,7168,0.28431999683380127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,10240,0.34969601035118103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,8192,0.28566399216651917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,12288,0.41305598616600037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,4096,0.17084799706935883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,5120,0.19059200584888458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,3072,0.1271039992570877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,2560,0.11507199704647064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,3584,0.1425279974937439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,2048,0.09612800180912018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,16384,0.5401920080184937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,1536,0.07622399926185608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,1024,0.058559998869895935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,512,0.03363199904561043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,256,0.023296000435948372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,128,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,12288,0.3010239899158478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,8192,0.21068799495697021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,10240,0.2560639977455139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,7168,0.18799999356269836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,16384,0.39158400893211365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,4096,0.1196800023317337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,5120,0.14233599603176117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,3584,0.10819199681282043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,3072,0.09839999675750732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,65536,2.0762240886688232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,2048,0.07225599884986877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,2560,0.08451200276613235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,1536,0.06035200133919716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,1024,0.0459199994802475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,512,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,128,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,256,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,65536,2.4409921169281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,8192,0.16592000424861908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,10240,0.2035519927740097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,12288,0.24134400486946106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,7168,0.14662399888038635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,4096,0.08979199826717377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,5120,0.10950399935245514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,3072,0.07129599899053574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,16384,0.31539198756217957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,3584,0.08211199939250946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,2048,0.05337600037455559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,2560,0.06220800057053566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,65536,1.4966720342636108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,1024,0.033055998384952545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,1536,0.04320000112056732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,512,0.023903999477624893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,128,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,256,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,7168,0.14662399888038635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,12288,0.2406720072031021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,10240,0.20310400426387787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,8192,0.1650560051202774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,3584,0.08102399855852127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,4096,0.0899519994854927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,5120,0.10892800241708755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,16384,0.3148159980773926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,3072,0.07103999704122543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,2048,0.052319999784231186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,1536,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,2560,0.062144000083208084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,512,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,1024,0.03347200155258179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,256,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,128,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,65536,1.2138240337371826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,12288,0.24393600225448608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,10240,0.20310400426387787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,8192,0.13471999764442444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,7168,0.12505599856376648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,5120,0.09414400160312653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,4096,0.07916799932718277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,16384,0.2433599978685379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,3584,0.07436800003051758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,3072,0.06588800251483917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,2048,0.05145600065588951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,512,0.020640000700950623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,1024,0.036320000886917114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,1536,0.04092799872159958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,2560,0.058400001376867294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,128,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,256,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,10240,0.13843199610710144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,12288,0.16121600568294525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,8192,0.11552000045776367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,16384,0.20678399503231049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,5120,0.08044800162315369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,4096,0.06780800223350525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,7168,0.10419200360774994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,3584,0.061503998935222626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,65536,1.2140159606933594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,3072,0.057631999254226685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,2560,0.05027199909090996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,65536,0.9019520282745361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,2048,0.045504000037908554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,1024,0.02751999907195568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,1536,0.03852799907326698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,512,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,128,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,256,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,8192,0.1151999980211258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,7168,0.07804799824953079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,10240,0.10566399991512299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,12288,0.12412799894809723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,16384,0.1616639941930771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,3584,0.05689600110054016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,4096,0.06332799792289734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,5120,0.07654400169849396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,3072,0.050335999578237534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,2560,0.035392001271247864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,2048,0.030688000842928886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,1024,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,512,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,1536,0.03248000144958496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,256,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,128,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,12288,0.12361600250005722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,8192,0.08684799820184708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,10240,0.10531199723482132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,16384,0.16118399798870087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,65536,0.7514560222625732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,5120,0.058720000088214874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,3584,0.04435199871659279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,3072,0.04025600105524063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,4096,0.048767998814582825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,7168,0.077504001557827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,2560,0.035071998834609985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,1536,0.02566399984061718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,1024,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,2048,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,512,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,65536,0.6109439730644226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,256,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,128,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,12288,0.07760000228881836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,8192,0.05958399921655655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,7168,0.05475199967622757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,10240,0.07494399696588516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,16384,0.09670399874448776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,4096,0.03516799956560135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,5120,0.04121600091457367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,3584,0.03161599859595299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,3072,0.02908799983561039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,2560,0.035840000957250595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,512,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,2048,0.030400000512599945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,1024,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,1536,0.02582399919629097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,65536,0.589024007320404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,256,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,128,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,8192,0.04064000025391579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,12288,0.052480001002550125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,7168,0.0453759990632534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,10240,0.04758400097489357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,16384,0.06415999680757523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,3584,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,4096,0.024992000311613083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,5120,0.02879999950528145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,3072,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,2560,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,2048,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,1536,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,1024,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,512,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,256,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,65536,0.5215039849281311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,12288,0.04198399931192398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,8192,0.033376000821590424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,10240,0.03702399879693985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,7168,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,16384,0.04726399853825569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,5120,0.0398080013692379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,4096,0.033663999289274216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,3584,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,2560,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,3072,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,2048,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,1536,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,512,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,1024,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,65536,0.20764799416065216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,12288,0.038975998759269714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,10240,0.033376000821590424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,7168,0.02924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,8192,0.03331200033426285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,16384,0.04630399867892265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,3072,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,4096,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,3584,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,5120,0.024320000782608986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,2560,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,1024,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,2048,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,512,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,1536,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,256,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,65536,0.12467200309038162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,5120,0.8023359775543213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,12288,1.8626879453659058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,10240,1.5680639743804932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,3584,0.5791360139846802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,3072,0.5031359791755676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,2560,0.4275520145893097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,2048,0.35395199060440063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,4096,0.653984010219574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,65536,0.1189119964838028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,7168,1.1056640148162842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,1536,0.2797439992427826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,1024,0.20390400290489197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,256,0.07126399874687195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,512,0.11052799969911575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,8192,1.2528959512710571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,128,0.06140799820423126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,12288,0.4707840085029602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,7168,0.2855679988861084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,8192,0.32204800844192505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,10240,0.3983359932899475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,16384,0.6259199976921082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,5120,0.2093760073184967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,4096,0.1712000072002411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,3072,0.1329279989004135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,2560,0.11478400230407715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,3584,0.15331199765205383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,1536,0.07472000271081924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,1024,0.054207999259233475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,2048,0.09724800288677216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,256,0.024064000695943832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,512,0.03372799977660179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,128,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,12288,0.35574400424957275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,10240,0.2993279993534088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,16384,2.472320079803467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,8192,0.24297599494457245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,4096,0.13116799294948578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,5120,0.15958400070667267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,7168,0.21526400744915009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,16384,0.46860799193382263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,3584,0.11820799857378006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,65536,2.4460160732269287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,3072,0.10281600058078766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,2560,0.0891840010881424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,2048,0.07507199794054031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,1536,0.06032000109553337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,512,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,1024,0.044415999203920364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,128,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,256,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,12288,0.3006080090999603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,8192,0.21241599321365356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,10240,0.2569600045681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,7168,0.18812799453735352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,16384,0.3924799859523773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,4096,0.1199679970741272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,5120,0.14556799829006195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,3584,0.09932799637317657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,2560,0.07929600030183792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,3072,0.08777599781751633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,1536,0.05116799846291542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,2048,0.0655680000782013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,256,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,512,0.02505600079894066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,1024,0.03807999938726425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,128,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,12288,0.24240000545978546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,8192,0.16492800414562225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,10240,0.2030079960823059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,16384,0.31676799058914185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,5120,0.10896000266075134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,7168,0.1470080018043518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,4096,0.0904960036277771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,3584,0.08150400221347809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,65536,1.5731199979782104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,3072,0.07116799801588058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,2560,0.062144000083208084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,2048,0.05100800096988678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,1536,0.04294399917125702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,512,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,256,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,1024,0.0318400003015995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,128,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,12288,0.2415039986371994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,8192,0.1658879965543747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,10240,0.20259200036525726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,65536,1.8434560298919678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,16384,0.3154880106449127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,4096,0.08963199704885483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,5120,0.10940799862146378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,7168,0.1472640037536621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,3584,0.0801599994301796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,1536,0.043935999274253845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,2560,0.06128000095486641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,2048,0.05331199988722801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,1024,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,3072,0.07094399631023407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,128,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,256,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,512,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,65536,1.2278720140457153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,8192,0.14368000626564026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,10240,0.17292800545692444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,12288,0.16128000617027283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,7168,0.1029760017991066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,16384,0.20678399503231049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,65536,1.2144960165023804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,5120,0.08089599758386612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,3072,0.06531199812889099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,4096,0.06745599955320358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,3584,0.06201599910855293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,2560,0.04841599985957146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,128,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,1536,0.034752000123262405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,512,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,1024,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,2048,0.04156799986958504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,256,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,7168,0.07968000322580338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,12288,0.12956799566745758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,10240,0.10838399827480316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,8192,0.08752000331878662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,16384,0.17136000096797943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,5120,0.05920000001788139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,4096,0.04915200173854828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,3584,0.04508800059556961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,3072,0.04012800008058548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,2560,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,1024,0.020896000787615776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,1536,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,2048,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,512,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,256,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,128,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,65536,0.7513279914855957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,12288,0.12464000284671783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,16384,0.16201600432395935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,8192,0.08707199990749359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,10240,0.10620799660682678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,7168,0.077504001557827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,4096,0.04956800118088722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,5120,0.05951999872922897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,2560,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,65536,0.6424319744110107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,3072,0.04028800129890442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,3584,0.045504000037908554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,1536,0.026976000517606735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,2048,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,256,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,1024,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,512,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,128,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,12288,0.1244800016283989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,8192,0.088128000497818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,10240,0.10505600273609161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,7168,0.07779199630022049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,16384,0.16195200383663177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,5120,0.06038400158286095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,4096,0.04879999905824661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,3584,0.04460800066590309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,3072,0.04073600098490715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,65536,0.6103360056877136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,2560,0.03494400158524513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,2048,0.03049599938094616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,1536,0.02691200003027916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,512,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,1024,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,128,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,256,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,8192,0.07705599814653397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,12288,0.10582400113344193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,7168,0.0689919963479042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,10240,0.09174399822950363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,16384,0.1375039964914322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,5120,0.05366399884223938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,2560,0.034752000123262405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,4096,0.04601600021123886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,3584,0.04153599962592125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,3072,0.03964800015091896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,65536,0.5861759781837463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,2048,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,1536,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,512,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,256,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,1024,0.02070399932563305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,128,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,7168,0.07718399912118912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,12288,0.06943999975919724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,8192,0.04918399825692177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,10240,0.07494399696588516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,16384,0.09008000046014786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,5120,0.057312000542879105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,3584,0.027615999802947044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,4096,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,3072,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,2560,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,1024,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,512,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,1536,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,2048,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,256,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,128,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,65536,0.4918079972267151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,12288,0.06643199920654297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,10240,0.05875200033187866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,8192,0.04870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,16384,0.07913599908351898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,5120,0.0344959981739521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,7168,0.04416000097990036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,3584,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,65536,0.4222719967365265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,4096,0.029343999922275543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,3072,0.024607999250292778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,2560,0.02304000034928322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,2048,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,1536,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,512,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,1024,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,256,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,12288,0.05452800169587135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,10240,0.04931199923157692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,8192,0.042080000042915344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,7168,0.044544000178575516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,16384,0.06694400310516357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,5120,0.029503999277949333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,4096,0.025151999667286873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,3584,0.03155200183391571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,3072,0.028224000707268715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,2560,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,2048,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,1536,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,512,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,1024,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,256,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,128,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,65536,0.24460799992084503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,12288,0.04323200136423111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,10240,0.03872000053524971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,8192,0.03340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,7168,0.031647998839616776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,16384,0.056063998490571976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,5120,0.02691200003027916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,4096,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,3584,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,3072,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,2560,0.025280000641942024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,2048,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,1024,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,1536,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,65536,0.2093760073184967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,12288,0.040672000497579575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,8192,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,10240,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,7168,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,16384,0.04896000027656555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,4096,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,5120,0.02486399933695793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,3584,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,3072,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,65536,0.1326719969511032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,2560,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,1024,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,2048,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,1536,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,256,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,12288,0.029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,7168,0.02364799939095974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,8192,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,10240,0.02659199945628643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,16384,0.034432001411914825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,3584,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,4096,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,3072,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,5120,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,65536,0.0950080007314682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,2560,0.016767999157309532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,1024,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,2048,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,1536,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,128,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,65536,0.0756480023264885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,5120,0.6072319746017456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,7168,0.8312320113182068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,4096,0.49881601333618164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,8192,0.9487680196762085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,2560,0.32604798674583435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,10240,1.177183985710144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,3584,0.4413439929485321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,2048,0.27113598585128784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,1024,0.15785600244998932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,3072,0.3819200098514557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,12288,1.4036799669265747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,512,0.18483200669288635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,1536,0.21475200355052948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,128,0.04944000020623207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,256,0.06032000109553337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,12288,0.3568960130214691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,7168,0.2160319983959198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,8192,0.2441280037164688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,10240,0.3023679852485657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,16384,0.4708479940891266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,3072,0.10284800082445145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,5120,0.15955199301242828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,3584,0.11740799993276596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,2560,0.08857599645853043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,16384,1.8582079410552979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,4096,0.13177600502967834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,1536,0.14156800508499146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,1024,0.10316800326108932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,2048,0.07462400197982788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,512,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,256,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,128,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,7168,0.17183999717235565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,8192,0.19148799777030945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,10240,0.23260800540447235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,12288,0.2754879891872406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,4096,0.10691200196743011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,3584,0.09785600006580353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,5120,0.12912000715732574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,65536,1.8390400409698486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,16384,0.3554239869117737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,3072,0.08828800171613693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,2560,0.07798399776220322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,2048,0.06569600105285645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,1024,0.03852799907326698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,1536,0.055743999779224396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,512,0.025599999353289604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,256,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,128,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,12288,0.24128000438213348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,8192,0.16521599888801575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,7168,0.14720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,5120,0.10883200168609619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,10240,0.20399999618530273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,3584,0.08137600123882294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,16384,0.31561601161956787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,4096,0.09059199690818787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,2048,0.1143999993801117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,3072,0.0708480030298233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,1536,0.08883199840784073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,2560,0.146464005112648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,1024,0.0331839993596077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,128,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,512,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,256,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,8192,0.13526399433612823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,10240,0.1621440052986145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,12288,0.18892799317836761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,16384,0.245728000998497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,65536,1.214143991470337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,3584,0.0721919983625412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,7168,0.12156800180673599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,5120,0.0947519987821579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,4096,0.07897599786520004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,3072,0.06534399837255478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,1536,0.03376000002026558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,2560,0.05798399820923805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,2048,0.05040000006556511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,1024,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,512,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,256,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,128,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,12288,0.16883200407028198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,8192,0.1204800009727478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,65536,1.3494399785995483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,10240,0.23824000358581543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,16384,0.31596800684928894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,65536,0.9053760170936584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,3584,0.08870399743318558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,5120,0.08419200032949448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,3072,0.05926400050520897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,4096,0.0716480016708374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,7168,0.10867200046777725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,2560,0.04879999905824661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,2048,0.04105599969625473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,1536,0.03468799963593483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,512,0.01881599985063076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,256,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,128,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,1024,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,12288,0.1682559996843338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,7168,0.10124800354242325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,8192,0.11484800279140472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,10240,0.14028799533843994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,16384,0.21859200298786163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,5120,0.07625599950551987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,4096,0.06268800050020218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,3584,0.05772799998521805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,3072,0.05087999999523163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,65536,0.8002240061759949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,2560,0.03596799820661545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,1536,0.041120000183582306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,1024,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,512,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,2048,0.03110400028526783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,256,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,128,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,12288,0.26256000995635986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,7168,0.07903999835252762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,10240,0.10550399869680405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,8192,0.08723200112581253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,16384,0.16208000481128693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,5120,0.0615679994225502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,4096,0.04966399818658829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,3072,0.040352001786231995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,2560,0.035551998764276505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,3584,0.04566400125622749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,2048,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,1024,0.020864000543951988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,1536,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,512,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,256,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,128,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,65536,0.610975980758667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,12288,0.12425599992275238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,10240,0.10540799796581268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,8192,0.08687999844551086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,16384,0.16182400286197662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,7168,0.07702399790287018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,5120,0.059487998485565186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,65536,1.4088000059127808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,3584,0.04460800066590309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,4096,0.04918399825692177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,3072,0.04025600105524063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,2560,0.03471999987959862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,2048,0.029920000582933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,1536,0.025919999927282333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,512,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,1024,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,256,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,128,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,8192,0.08873599767684937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,7168,0.07868800312280655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,10240,0.10585600137710571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,12288,0.09750399738550186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,16384,0.12358400225639343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,4096,0.04944000020623207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,5120,0.05984000116586685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,3584,0.04524800181388855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,3072,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,2560,0.03497600182890892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,2048,0.030719999223947525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,1536,0.02627200074493885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,1024,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,512,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,256,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,128,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,65536,0.6103360056877136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,12288,0.08479999750852585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,8192,0.06003199890255928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,10240,0.07276800274848938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,16384,0.10982400178909302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,65536,0.6101120114326477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,3584,0.031647998839616776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,4096,0.03497600182890892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,5120,0.04092799872159958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,3072,0.03999999910593033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,7168,0.053408000618219376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,2560,0.03532800078392029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,1024,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,1536,0.025728000327944756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,2048,0.030400000512599945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,512,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,256,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,128,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,12288,0.06812799721956253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,10240,0.07462400197982788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,8192,0.049215998500585556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,7168,0.04412800073623657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,16384,0.11321599781513214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,5120,0.05040000006556511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,65536,0.40934398770332336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,3072,0.03376000002026558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,3584,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,4096,0.040608000010252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,2560,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,2048,0.025887999683618546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,1536,0.017791999503970146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,1024,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,256,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,128,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,512,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,8192,0.04873599857091904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,7168,0.04419200122356415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,12288,0.05920000001788139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,10240,0.05475199967622757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,16384,0.06937599927186966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,5120,0.03440000116825104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,3584,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,3072,0.02566399984061718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,4096,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,65536,0.4220159947872162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,2048,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,2560,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,1536,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,1024,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,256,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,512,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,8192,0.03827200084924698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,10240,0.043935999274253845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,12288,0.04822399839758873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,7168,0.03446400165557861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,16384,0.05660799890756607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,5120,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,3584,0.03145600110292435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,4096,0.024159999564290047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,3072,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,2560,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,512,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,1024,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,256,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,1536,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,2048,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,65536,0.1929280012845993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,128,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,12288,0.03766399994492531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,8192,0.033344000577926636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,10240,0.03606399893760681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,7168,0.03094400092959404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,16384,0.04505600035190582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,5120,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,4096,0.03379200026392937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,65536,0.14444799721240997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,3072,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,3584,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,2560,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,2048,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,1024,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,1536,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,12288,0.03372799977660179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,8192,0.027615999802947044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,10240,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,16384,0.03907199949026108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,7168,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,5120,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,3584,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,4096,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,3072,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,2560,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,65536,0.09731200337409973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,2048,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,1536,0.01375999953597784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,1024,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,12288,0.027871999889612198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,8192,0.025567999109625816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,7168,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,10240,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,16384,0.03152000159025192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,65536,0.07753600180149078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,4096,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,5120,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,3584,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,3072,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,2048,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,2560,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,1536,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,1024,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,256,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,128,0.008128000423312187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,5120,0.40643200278282166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,8192,0.63155198097229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,12288,0.9319040179252625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,65536,0.06435199826955795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,16384,1.230463981628418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,3584,0.2948800027370453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,2048,0.1825920045375824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,2560,0.21609599888324738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,1536,0.13977600634098053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,4096,0.33209601044654846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,7168,0.5582079887390137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,3072,0.2582719922065735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,256,0.04156799986958504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,512,0.05951999872922897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,1024,0.09862399846315384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,128,0.036031998693943024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,10240,0.781823992729187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,12288,0.24009600281715393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,7168,0.14710399508476257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,16384,0.3152320086956024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,10240,0.20233599841594696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,5120,0.10921599715948105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,4096,0.08963199704885483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,8192,0.16543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,3584,0.08191999793052673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,3072,0.07062400132417679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,2560,0.06176000088453293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,2048,0.05097600072622299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,1024,0.030719999223947525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,1536,0.04153599962592125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,512,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,256,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,128,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,12288,0.1908479928970337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,7168,0.11398400366306305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,10240,0.18572799861431122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,8192,0.13100799918174744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,5120,0.08611200004816055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,3584,0.06364800035953522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,4096,0.06988800317049026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,3072,0.055424001067876816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,16384,0.24377599358558655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,2048,0.041152000427246094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,2560,0.04816000163555145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,65536,1.2163200378417969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,1536,0.033695999532938004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,512,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,1024,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,256,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,128,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,12288,0.20483200252056122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,8192,0.14444799721240997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,10240,0.17238399386405945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,16384,0.26787200570106506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,7168,0.12678399682044983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,5120,0.09667199850082397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,65536,0.8957440257072449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,3584,0.07056000083684921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,4096,0.0799039974808693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,2560,0.04867200180888176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,3072,0.06473600119352341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,2048,0.0488319993019104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,1024,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,1536,0.034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,256,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,128,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,512,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,12288,0.12495999783277512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,8192,0.08905600011348724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,7168,0.07785599678754807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,10240,0.10672000050544739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,16384,0.17609600722789764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,4096,0.049215998500585556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,3584,0.04492799937725067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,5120,0.05894400179386139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,65536,1.0139199495315552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,3072,0.04025600105524063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,2560,0.03532800078392029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,1536,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,2048,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,1024,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,512,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,128,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,256,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,12288,0.12694400548934937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,7168,0.07715199887752533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,8192,0.08752000331878662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,10240,0.110944002866745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,16384,0.16185599565505981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,4096,0.050016000866889954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,3584,0.044895999133586884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,5120,0.0605119988322258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,65536,0.6111680269241333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,3072,0.039712000638246536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,2560,0.034912001341581345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,1024,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,1536,0.025887999683618546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,2048,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,512,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,128,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,256,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,12288,0.12387199699878693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,8192,0.09004800021648407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,7168,0.07952000200748444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,10240,0.10569600015878677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,16384,0.16172799468040466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,5120,0.0607680007815361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,3072,0.03999999910593033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,3584,0.04447999969124794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,4096,0.04992000013589859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,2560,0.0350399985909462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,65536,0.6113600134849548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,2048,0.030400000512599945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,1536,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,512,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,256,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,1024,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,128,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,12288,0.08726400136947632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,8192,0.07529599964618683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,7168,0.06835199892520905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,10240,0.05967999994754791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,16384,0.08816000074148178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,5120,0.05344000086188316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,65536,0.6128640174865723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,4096,0.040352001786231995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,3584,0.03753599897027016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,3072,0.03331200033426285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,2560,0.028736000880599022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,2048,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,1536,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,256,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,512,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,1024,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,128,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,12288,0.06867200136184692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,7168,0.04464000090956688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,10240,0.05859199911355972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,8192,0.049056001007556915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,16384,0.088128000497818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,65536,0.4225600063800812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,4096,0.029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,3584,0.027648000046610832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,5120,0.034432001411914825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,3072,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,2048,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,2560,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,1536,0.01775999926030636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,512,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,1024,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,256,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,128,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,12288,0.08780799806118011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,7168,0.05532800033688545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,8192,0.04902400076389313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,10240,0.07436800003051758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,16384,0.11299200356006622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,65536,0.28806400299072266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,5120,0.049695998430252075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,4096,0.02969600073993206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,2560,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,3584,0.03638400137424469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,3072,0.033055998384952545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,2048,0.025728000327944756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,1536,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,512,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,1024,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,256,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,128,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,12288,0.06172800064086914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,7168,0.04383999854326248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,8192,0.04879999905824661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,10240,0.05462399870157242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,16384,0.07327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,65536,0.24486400187015533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,5120,0.034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,4096,0.029920000582933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,3584,0.02687999978661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,3072,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,2560,0.022207999601960182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,1536,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,2048,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,1024,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,256,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,512,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,12288,0.05612799897789955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,8192,0.041728001087903976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,7168,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,10240,0.048576001077890396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,16384,0.0676800012588501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,65536,0.231455996632576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,4096,0.034015998244285583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,5120,0.040991999208927155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,3584,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,2560,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,3072,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,2048,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,1536,0.01881599985063076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,512,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,1024,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,128,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,256,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,12288,0.04588799923658371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,8192,0.03888000175356865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,10240,0.04150399938225746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,7168,0.03510399907827377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,65536,0.21510399878025055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,16384,0.057312000542879105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,5120,0.027583999559283257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,4096,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,3072,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,3584,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,2560,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,2048,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,1024,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,512,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,1536,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,256,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,12288,0.04630399867892265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,8192,0.034015998244285583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,7168,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,65536,0.14486399292945862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,10240,0.03731200098991394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,16384,0.054336000233888626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,5120,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,4096,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,3072,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,3584,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,2560,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,2048,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,1536,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,1024,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,12288,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,10240,0.035999998450279236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,8192,0.03161599859595299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,7168,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,16384,0.04899200052022934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,65536,0.13289600610733032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,5120,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,4096,0.022207999601960182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,3584,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,3072,0.018079999834299088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,2560,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,2048,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,1536,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,256,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,12288,0.029152000322937965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,8192,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,7168,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,16384,0.033215999603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,10240,0.035360001027584076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,65536,0.08540800213813782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,3584,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,5120,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,4096,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,3072,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,2560,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,2048,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,1536,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,12288,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,10240,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,7168,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,8192,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,16384,0.030047999694943428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,65536,0.0817599967122078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,5120,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,3584,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,4096,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,2560,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,3072,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,2048,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,1536,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,512,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,256,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,1024,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,5120,0.45452800393104553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,65536,0.05407999828457832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,12288,1.158400058746338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,10240,0.9156479835510254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,4096,0.41523200273513794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,3072,0.2759360074996948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,2048,0.38659200072288513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,1536,0.2958720028400421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,3584,0.29737600684165955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,2560,0.48419201374053955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,7168,0.6369919776916504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,8192,0.7366719841957092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,512,0.09782399982213974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,256,0.04870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,128,0.03219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,1024,0.17894400656223297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,8192,0.19340799748897552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,10240,0.24806399643421173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,12288,0.2826879918575287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,7168,0.16582399606704712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,16384,0.35977599024772644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,4096,0.097120001912117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,5120,0.12703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,3072,0.07760000228881836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,3584,0.1695999950170517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,2560,0.0663359984755516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,2048,0.10060799866914749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,1536,0.0721919983625412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,1024,0.04732799902558327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,512,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,16384,1.5234880447387695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,256,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,128,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,8192,0.25516799092292786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,16384,0.2881599962711334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,7168,0.23107199370861053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,12288,0.38764798641204834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,10240,0.33129599690437317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,65536,1.5468480587005615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,5120,0.171424001455307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,3072,0.1048320010304451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,3584,0.1273919939994812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,2560,0.08755200356245041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,4096,0.14339199662208557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,2048,0.07273600250482559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,1536,0.05657599866390228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,1024,0.03619199991226196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,512,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,256,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,128,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,12288,0.20892800390720367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,8192,0.16140800714492798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,10240,0.19702400267124176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,7168,0.12649600207805634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,16384,0.21769599616527557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,65536,1.107807993888855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,5120,0.10527999699115753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,4096,0.08521600067615509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,3584,0.05584000051021576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,2560,0.05567999929189682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,2048,0.049247998744249344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,3072,0.06425599753856659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,512,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,1536,0.04012800008058548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,1024,0.03286400064826012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,256,0.015039999969303608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,128,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,12288,0.263264000415802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,10240,0.11686400324106216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,8192,0.1653120070695877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,7168,0.08339200168848038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,16384,0.3567039966583252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,5120,0.1128000020980835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,4096,0.09455999732017517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,3584,0.0854400023818016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,3072,0.07081600278615952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,65536,0.8349760174751282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,2560,0.059967998415231705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,2048,0.05023999884724617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,1536,0.03932800143957138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,1024,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,512,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,128,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,256,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,12288,0.24764800071716309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,8192,0.08796799927949905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,7168,0.1353919953107834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,10240,0.12508800625801086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,16384,0.16275200247764587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,5120,0.11446399986743927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,65536,0.7503359913825989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,4096,0.04934399947524071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,3584,0.081727996468544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,2560,0.059487998485565186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,3072,0.04064000025391579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,2048,0.030848000198602676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,1536,0.03791999816894531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,1024,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,512,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,256,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,128,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,12288,0.12396799772977829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,8192,0.14176000654697418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,7168,0.1225920021533966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,10240,0.10579200088977814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,16384,0.16143999993801117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,5120,0.04131200164556503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,4096,0.03558399900794029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,3584,0.0315839983522892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,3072,0.05711999908089638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,2560,0.04966399818658829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,2048,0.03920000046491623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,1536,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,1024,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,512,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,256,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,128,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,65536,0.6116480231285095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,12288,0.0875839963555336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,8192,0.07171200215816498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,10240,0.09276799857616425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,7168,0.08432000130414963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,16384,0.11334399878978729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,65536,0.613152027130127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,5120,0.0496320016682148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,4096,0.040832001715898514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,2048,0.02831999957561493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,3584,0.035999998450279236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,2560,0.02879999950528145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,3072,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,1536,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,1024,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,512,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,256,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,12288,0.12217599898576736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,8192,0.08921600133180618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,7168,0.07849600166082382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,10240,0.10758399963378906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,65536,0.4219839870929718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,5120,0.05974400043487549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,16384,0.09654399752616882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,4096,0.04838399961590767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,3584,0.04483199864625931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,3072,0.03830400109291077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,2560,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,2048,0.02876799926161766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,1536,0.02223999984562397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,1024,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,256,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,512,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,128,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,12288,0.06940799951553345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,7168,0.05648000165820122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,10240,0.07433599978685379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,65536,0.33772799372673035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,8192,0.08380799740552902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,16384,0.08915200084447861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,5120,0.04649600014090538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,4096,0.04761600121855736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,3584,0.04195199906826019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,3072,0.0363520011305809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,2048,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,2560,0.028575999662280083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,512,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,1024,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,1536,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,256,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,128,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,12288,0.06335999816656113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,7168,0.06800000369548798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,8192,0.07427199929952621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,10240,0.06063999980688095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,65536,0.2452159970998764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,16384,0.07744000107049942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,5120,0.05475199967622757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,3584,0.03654399886727333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,4096,0.03852799907326698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,3072,0.0326399989426136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,2560,0.02687999978661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,2048,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,1536,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,1024,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,512,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,256,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,128,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,12288,0.048767998814582825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,10240,0.042527999728918076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,65536,0.2451840043067932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,16384,0.05663999915122986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,8192,0.038975998759269714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,7168,0.047807998955249786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,5120,0.040352001786231995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,4096,0.03446400165557861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,3072,0.02831999957561493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,3584,0.02876799926161766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,2048,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,2560,0.02521600015461445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,1536,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,128,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,512,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,256,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,1024,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,12288,0.05289600044488907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,10240,0.03916800022125244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,16384,0.049375999718904495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,7168,0.03468799963593483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,8192,0.03468799963593483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,65536,0.1685439944267273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,3584,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,4096,0.026079999282956123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,5120,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,2560,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,2048,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,3072,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,1024,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,1536,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,12288,0.0395519994199276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,8192,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,65536,0.1605760008096695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,7168,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,10240,0.03411199897527695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,16384,0.045632001012563705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,5120,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,4096,0.03411199897527695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,3584,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,2560,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,3072,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,2048,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,1536,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,128,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,12288,0.03161599859595299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,8192,0.02783999964594841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,7168,0.02582399919629097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,10240,0.03017600066959858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,16384,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,65536,0.13436800241470337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,5120,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,4096,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,3584,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,3072,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,2560,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,2048,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,1536,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,12288,0.027807999402284622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,7168,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,10240,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,8192,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,16384,0.03110400028526783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,65536,0.07436800003051758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,5120,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,4096,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,3584,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,2560,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,3072,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,2048,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,1536,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,128,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,12288,0.026784000918269157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,10240,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,8192,0.023104000836610794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,16384,0.029791999608278275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,7168,0.022431999444961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,65536,0.06639999896287918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,4096,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,3584,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,5120,0.020896000787615776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,2048,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,2560,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,3072,0.017983999103307724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,1536,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,65536,0.046911999583244324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,5120,0.2083200067281723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,8192,0.3208959996700287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,7168,0.2842240035533905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,12288,0.47062399983406067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,10240,0.39718401432037354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,4096,0.17177599668502808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,16384,0.6215680241584778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,3072,0.13251200318336487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,2048,0.09324800223112106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,1536,0.07235199958086014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,512,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,1024,0.05299200117588043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,128,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,256,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,2560,0.11187200248241425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,3584,0.15132799744606018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,12288,0.12777599692344666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,7168,0.07977599650621414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,8192,0.08975999802350998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,5120,0.05955199897289276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,10240,0.10915199667215347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,16384,0.1685439944267273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,4096,0.05097600072622299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,2048,0.029632000252604485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,2560,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,3584,0.04416000097990036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,1536,0.02691200003027916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,3072,0.040063999593257904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,512,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,1024,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,128,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,256,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,7168,0.07753600180149078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,10240,0.1056319996714592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,16384,0.16102400422096252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,12288,0.1258240044116974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,8192,0.0870399996638298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,3584,0.044704001396894455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,65536,0.6110720038414001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,4096,0.04992000013589859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,5120,0.059007998555898666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,3072,0.04025600105524063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,2560,0.03481600061058998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,1024,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,1536,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,512,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,256,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,2048,0.03017600066959858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,128,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,12288,0.12364800274372101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,7168,0.07878399640321732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,10240,0.10534399747848511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,8192,0.08857599645853043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,16384,0.16128000617027283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,5120,0.05942400172352791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,3584,0.04195199906826019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,4096,0.0496320016682148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,3072,0.040608000010252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,2560,0.03433600068092346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,1536,0.024960000067949295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,2048,0.029600000008940697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,512,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,1024,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,256,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,128,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,12288,0.0878399983048439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,65536,0.5883520245552063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,10240,0.05894400179386139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,8192,0.07308799773454666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,16384,0.1133119985461235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,65536,0.6111999750137329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,7168,0.04419200122356415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,5120,0.05056000128388405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,3584,0.03587200120091438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,4096,0.04089599847793579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,3072,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,2560,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,2048,0.024032000452280045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,1536,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,1024,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,512,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,256,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,128,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,12288,0.08710400015115738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,7168,0.0647680014371872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,10240,0.0745600014925003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,8192,0.04879999905824661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,16384,0.11324799805879593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,65536,0.3160640001296997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,5120,0.04806400090456009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,4096,0.04102399945259094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,3072,0.03292800113558769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,3584,0.035679999738931656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,2560,0.028063999488949776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,1536,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,2048,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,1024,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,512,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,256,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,128,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,12288,0.0870399996638298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,7168,0.05526399984955788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,10240,0.07257600128650665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,8192,0.0490880012512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,16384,0.11260800063610077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,5120,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,65536,0.28118398785591125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,4096,0.040672000497579575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,3584,0.035232000052928925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,3072,0.03283200040459633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,2560,0.027744000777602196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,2048,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,1024,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,512,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,256,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,1536,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,128,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,12288,0.05564799904823303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,8192,0.041919998824596405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,10240,0.049247998744249344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,7168,0.04499199986457825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,16384,0.06710399687290192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,65536,0.42127999663352966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,5120,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,2560,0.024032000452280045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,4096,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,3072,0.028192000463604927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,3584,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,2048,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,1536,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,1024,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,512,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,128,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,256,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,8192,0.03961599990725517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,7168,0.0360959991812706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,12288,0.050335999578237534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,65536,0.20854400098323822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,10240,0.04588799923658371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,16384,0.05936000123620033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,2048,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,5120,0.028384000062942505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,4096,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,3072,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,2560,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,3584,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,1536,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,128,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,512,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,12288,0.07097599655389786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,65536,0.16288000345230103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,8192,0.049855999648571014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,7168,0.045951999723911285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,10240,0.06060799956321716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,16384,0.06092799827456474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,4096,0.03433600068092346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,5120,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,2560,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,3584,0.03046399913728237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,3072,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,2048,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,1536,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,1024,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,512,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,128,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,12288,0.04527999833226204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,8192,0.03564799949526787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,10240,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,7168,0.03455999866127968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,65536,0.17027199268341064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,16384,0.05132799968123436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,5120,0.027871999889612198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,2048,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,2560,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,3584,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,3072,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,4096,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,1536,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,256,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,512,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,1024,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,12288,0.04383999854326248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,8192,0.034432001411914825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,10240,0.03929600119590759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,65536,0.1342719942331314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,16384,0.05321599915623665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,7168,0.03340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,5120,0.02703999914228916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,4096,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,3072,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,3584,0.029888000339269638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,2048,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,1536,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,1024,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,12288,0.03759999945759773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,7168,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,8192,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,65536,0.13366399705410004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,10240,0.0342399999499321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,16384,0.045823998749256134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,5120,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,4096,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,3072,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,2560,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,3584,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,2048,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,1536,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,1024,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,12288,0.04047999903559685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,7168,0.029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,8192,0.028095999732613564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,65536,0.09692800045013428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,10240,0.034304000437259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,16384,0.04793599992990494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,5120,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,4096,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,3584,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,2560,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,3072,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,2048,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,512,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,1536,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,12288,0.037856001406908035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,8192,0.030368000268936157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,10240,0.03411199897527695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,7168,0.028063999488949776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,16384,0.03244800120592117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,65536,0.09571199864149094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,5120,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,4096,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,3584,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,3072,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,2560,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,12288,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,65536,0.08057600259780884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,10240,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,7168,0.022336000576615334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,8192,0.023104000836610794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,16384,0.028704000636935234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,4096,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,5120,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,3584,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,3072,0.018079999834299088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,256,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,12288,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,65536,0.053119998425245285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,16384,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,8192,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,7168,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,5120,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,10240,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,3584,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,3072,0.017983999103307724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,4096,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,1536,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,512,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,256,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,65536,0.04825599864125252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,5120,0.21171200275421143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,7168,0.2863680124282837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,16384,0.620415985584259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,3584,0.30614399909973145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,4096,0.17244799435138702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,3072,0.2552320063114166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,8192,0.3250559866428375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,2048,0.16838400065898895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,1536,0.13072000443935394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,1024,0.08550400286912918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,2560,0.21475200355052948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,10240,0.40828800201416016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,128,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,12288,0.4781759977340698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,256,0.026079999282956123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,512,0.04374400153756142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,5120,0.10473600029945374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,12288,0.23398399353027344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,8192,0.15695999562740326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,7168,0.13926400244235992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,10240,0.196383997797966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,16384,0.31836798787117004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,4096,0.0867839977145195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,3072,0.04028800129890442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,2560,0.05398400127887726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,3584,0.04524800181388855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,1536,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,2048,0.04339199885725975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,512,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,128,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,1024,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,256,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,12288,0.13555200397968292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,8192,0.0872960016131401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,7168,0.07737600058317184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,10240,0.1157120019197464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,5120,0.05852799862623215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,16384,0.17347200214862823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,4096,0.04944000020623207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,65536,0.6116480231285095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,3584,0.044704001396894455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,3072,0.03977600112557411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,2560,0.048128001391887665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,2048,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,1536,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,1024,0.02412799932062626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,512,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,256,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,128,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,12288,0.12377600371837616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,8192,0.1263359934091568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,7168,0.11292800307273865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,65536,0.6130239963531494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,16384,0.16201600432395935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,10240,0.10579200088977814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,3584,0.06015999987721443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,2560,0.045024000108242035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,5120,0.06278400123119354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,4096,0.06752000004053116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,2048,0.03763199970126152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,3072,0.04435199871659279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,256,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,1024,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,1536,0.029503999277949333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,512,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,128,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,12288,0.08819200098514557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,10240,0.09961599856615067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,8192,0.09542399644851685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,7168,0.0854720026254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,16384,0.11747200042009354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,65536,0.6104959845542908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,5120,0.06294400244951248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,4096,0.053119998425245285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,3072,0.035232000052928925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,3584,0.0496320016682148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,2560,0.037408001720905304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,2048,0.03001599945127964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,1536,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,1024,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,512,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,128,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,256,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,12288,0.11356800049543381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,65536,0.422432005405426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,7168,0.08476799726486206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,16384,0.1133119985461235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,8192,0.07558400183916092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,10240,0.07507199794054031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,5120,0.05571199953556061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,4096,0.05145600065588951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,2560,0.03545600175857544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,3072,0.03984000161290169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,3584,0.0469760000705719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,2048,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,1536,0.023840000852942467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,512,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,1024,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,256,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,128,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,12288,0.08707199990749359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,8192,0.06172800064086914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,10240,0.07449600100517273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,7168,0.054336000233888626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,16384,0.11046399921178818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,65536,0.42214399576187134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,5120,0.056384000927209854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,4096,0.04668800160288811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,2560,0.03254399821162224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,3584,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,3072,0.03641600161790848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,2048,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,1536,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,256,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,512,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,1024,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,12288,0.056063998490571976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,8192,0.04255999997258186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,10240,0.0488319993019104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,7168,0.04479999840259552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,65536,0.420991986989975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,16384,0.06931199878454208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,5120,0.03488000109791756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,3584,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,4096,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,3072,0.02703999914228916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,2048,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,2560,0.024992000311613083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,1536,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,1024,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,256,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,512,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,128,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,12288,0.05350400134921074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,8192,0.051263999193906784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,10240,0.045504000037908554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,65536,0.21488000452518463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,7168,0.045823998749256134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,16384,0.06444799900054932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,5120,0.03561599925160408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,3584,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,2560,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,3072,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,2048,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,4096,0.02921600081026554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,1536,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,128,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,1024,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,512,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,12288,0.07116799801588058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,8192,0.049695998430252075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,7168,0.044704001396894455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,65536,0.160288006067276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,10240,0.05990400165319443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,16384,0.0605119988322258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,5120,0.034591998904943466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,2560,0.025151999667286873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,2048,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,4096,0.03379200026392937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,3584,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,3072,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,1536,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,1024,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,512,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,256,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,12288,0.04467200115323067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,65536,0.1671999990940094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,8192,0.03907199949026108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,7168,0.03519999980926514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,10240,0.04012800008058548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,16384,0.05305600166320801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,4096,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,3584,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,2560,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,5120,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,2048,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,3072,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,1024,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,1536,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,128,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,12288,0.04233599826693535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,8192,0.03126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,65536,0.13091200590133667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,7168,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,10240,0.0350399985909462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,16384,0.046431999653577805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,5120,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,4096,0.02179200015962124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,3584,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,2560,0.02502400055527687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,3072,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,2048,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,1536,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,1024,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,12288,0.035360001027584076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,65536,0.1305599957704544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,8192,0.02812799997627735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,7168,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,16384,0.046271998435258865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,10240,0.032287999987602234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,5120,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,4096,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,3584,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,2560,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,3072,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,1536,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,12288,0.032896000891923904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,7168,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,8192,0.028351999819278717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,10240,0.030400000512599945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,16384,0.03852799907326698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,65536,0.09852799773216248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,4096,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,5120,0.02271999977529049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,3584,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,3072,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,2560,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,1024,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,1536,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,12288,0.032255999743938446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,8192,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,65536,0.07878399640321732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,7168,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,10240,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,16384,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,5120,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,4096,0.02070399932563305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,3584,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,2560,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,3072,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,2048,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,12288,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,65536,0.06537599861621857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,10240,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,8192,0.022752000018954277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,16384,0.02940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,7168,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,5120,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,4096,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,3584,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,3072,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,2560,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,1536,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,1024,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,512,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,256,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,12288,0.026048000901937485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,65536,0.047520000487565994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,10240,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,8192,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,16384,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,7168,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,4096,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,3584,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,3072,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,5120,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,2560,0.016448000445961952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,65536,0.04771199822425842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,7168,0.308896005153656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,5120,0.23059199750423431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,8192,0.3479680120944977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,12288,0.5225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,10240,0.42211198806762695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,2560,0.25334399938583374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,3072,0.2783359885215759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,3584,0.3535040020942688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,4096,0.3816640079021454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,16384,0.6578879952430725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,1536,0.15027199685573578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,2048,0.17696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,512,0.05587200075387955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,128,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,1024,0.08953599631786346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,256,0.028831999748945236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,12288,0.25910401344299316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,7168,0.15004800260066986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,5120,0.06831999868154526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,8192,0.1582079976797104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,10240,0.2189760059118271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,16384,0.18188799917697906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,4096,0.09596800059080124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,3584,0.08563199639320374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,2048,0.048287998884916306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,3072,0.06934399902820587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,2560,0.06060799956321716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,1536,0.039903998374938965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,1024,0.026079999282956123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,256,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,512,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,128,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,12288,0.12345600128173828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,8192,0.09180799871683121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,16384,0.19196799397468567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,7168,0.08956799656152725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,10240,0.12390399724245071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,5120,0.06483200192451477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,2560,0.05167999863624573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,4096,0.05049600079655647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,3584,0.04508800059556961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,3072,0.04137599840760231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,65536,0.6444159746170044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,1024,0.02412799932062626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,256,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,2048,0.03094400092959404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,512,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,1536,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,128,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,12288,0.12275200337171555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,8192,0.13519999384880066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,7168,0.08511999994516373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,10240,0.11363200098276138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,16384,0.16172799468040466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,4096,0.07391999661922455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,5120,0.0838719978928566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,3072,0.048128001391887665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,3584,0.05411199852824211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,2560,0.050016000866889954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,2048,0.03936000168323517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,1536,0.030912000685930252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,1024,0.02304000034928322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,512,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,256,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,128,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,12288,0.09161599725484848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,10240,0.07779199630022049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,8192,0.10361599922180176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,65536,0.6865280270576477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,16384,0.12057600170373917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,65536,0.6117119789123535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,7168,0.09657599776983261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,5120,0.07030399888753891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,4096,0.06111999973654747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,3072,0.04831999912858009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,3584,0.05660799890756607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,2560,0.04076800122857094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,1024,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,2048,0.03280000016093254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,1536,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,512,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,128,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,256,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,12288,0.0865280032157898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,8192,0.09728000313043594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,7168,0.0907519981265068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,10240,0.0772159993648529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,16384,0.11347199976444244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,5120,0.0713919997215271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,4096,0.056832000613212585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,3072,0.03936000168323517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,3584,0.05644800141453743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,65536,0.31593599915504456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,2560,0.03267199918627739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,2048,0.03251200169324875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,1536,0.026655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,1024,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,512,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,256,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,128,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,12288,0.08627200126647949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,8192,0.06080000102519989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,10240,0.07363200187683105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,16384,0.11395200341939926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,7168,0.054336000233888626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,65536,0.4228479862213135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,5120,0.05987200140953064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,4096,0.050175998359918594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,3072,0.04044799879193306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,3584,0.04032000154256821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,2560,0.0350399985909462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,2048,0.029184000566601753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,1536,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,1024,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,512,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,128,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,256,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,12288,0.054816000163555145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,10240,0.04841599985957146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,7168,0.043487999588251114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,8192,0.04412800073623657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,16384,0.06937599927186966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,65536,0.2096959948539734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,5120,0.03471999987959862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,4096,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,3584,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,3072,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,2560,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,2048,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,1536,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,1024,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,512,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,128,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,256,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,65536,0.4119040071964264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,12288,0.056223999708890915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,16384,0.06614399701356888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,8192,0.05951999872922897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,7168,0.050016000866889954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,10240,0.04835199937224388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,5120,0.03811199963092804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,4096,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,3584,0.029600000008940697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,2560,0.023104000836610794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,2048,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,3072,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,1536,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,1024,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,512,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,128,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,256,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,12288,0.0695360004901886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,65536,0.16102400422096252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,10240,0.04368000105023384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,16384,0.06080000102519989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,7168,0.044415999203920364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,8192,0.04960000142455101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,5120,0.034623999148607254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,4096,0.03411199897527695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,3584,0.03110400028526783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,3072,0.02783999964594841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,2048,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,2560,0.025151999667286873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,1536,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,1024,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,512,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,256,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,128,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,12288,0.047040000557899475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,65536,0.144896000623703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,10240,0.03977600112557411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,8192,0.04041599854826927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,16384,0.05321599915623665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,7168,0.035840000957250595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,3072,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,4096,0.02908799983561039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,5120,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,2560,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,3584,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,2048,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,1536,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,128,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,256,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,512,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,12288,0.03840000182390213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,7168,0.02969600073993206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,65536,0.12963199615478516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,8192,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,10240,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,16384,0.0459199994802475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,5120,0.02598400041460991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,2560,0.024607999250292778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,4096,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,3584,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,3072,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,2048,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,1536,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,1024,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,512,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,128,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,256,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,12288,0.033695999532938004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,65536,0.13123199343681335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,8192,0.027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,7168,0.027807999402284622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,10240,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,16384,0.04390399903059006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,5120,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,4096,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,3584,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,3072,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,2560,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,2048,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,1536,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,512,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,1024,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,128,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,256,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,65536,0.10313600301742554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,12288,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,8192,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,16384,0.03612799942493439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,10240,0.028192000463604927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,7168,0.02348800003528595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,4096,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,5120,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,3072,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,3584,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,2560,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,2048,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,1536,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,256,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,512,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,12288,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,65536,0.0785600021481514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,8192,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,10240,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,7168,0.022304000332951546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,16384,0.029600000008940697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,5120,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,4096,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,3584,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,3072,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,2048,0.015039999969303608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,2560,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,1536,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,512,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,12288,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,65536,0.05936000123620033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,7168,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,8192,0.022207999601960182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,16384,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,10240,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,5120,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,2560,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,3584,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,2048,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,3072,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,4096,0.020896000787615776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,1536,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,512,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,128,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,256,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,12288,0.02659199945628643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,65536,0.046751998364925385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,8192,0.026464000344276428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,10240,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,7168,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,16384,0.028991999104619026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,5120,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,4096,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,3072,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,3584,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,2560,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,2048,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,1536,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,256,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,65536,0.052960000932216644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,5120,0.13577599823474884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,7168,0.18572799861431122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,8192,0.21007999777793884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,12288,0.3083840012550354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,10240,0.25865599513053894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,4096,0.11123199760913849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,3584,0.10022400319576263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,2560,0.07507199794054031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,1024,0.039712000638246536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,2048,0.06307200342416763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,1536,0.05104000121355057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,512,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,3072,0.0867839977145195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,256,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,128,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,12288,0.1035199984908104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,10240,0.08975999802350998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,8192,0.07177600264549255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,7168,0.06272000074386597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,16384,0.4049279987812042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,5120,0.04732799902558327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,16384,0.13315199315547943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,4096,0.04032000154256821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,3584,0.03651199862360954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,2560,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,2048,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,3072,0.03167999908328056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,1536,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,1024,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,512,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,128,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,256,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,12288,0.08483199775218964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,7168,0.05273599922657013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,8192,0.06054399907588959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,16384,0.10950399935245514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,10240,0.07225599884986877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,5120,0.04028800129890442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,3584,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,4096,0.03388800099492073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,65536,0.5693759918212891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,2560,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,3072,0.027712000533938408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,2048,0.02191999927163124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,1536,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,512,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,1024,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,256,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,12288,0.08448000252246857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,8192,0.06159999966621399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,7168,0.05462399870157242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,10240,0.07478400319814682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,16384,0.11270400136709213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,65536,0.4005120098590851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,5120,0.04662400111556053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,3584,0.03683200106024742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,3072,0.03126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,4096,0.03376000002026558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,2560,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,2048,0.024032000452280045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,1536,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,512,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,128,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,256,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,1024,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,12288,0.05484800040721893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,7168,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,10240,0.0490880012512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,8192,0.04224000126123428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,16384,0.06703999638557434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,65536,0.4009920060634613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,5120,0.03968000039458275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,3584,0.023072000592947006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,4096,0.033215999603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,2048,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,2560,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,3072,0.027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,1536,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,1024,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,256,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,512,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,12288,0.05305600166320801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,8192,0.04224000126123428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,7168,0.043807998299598694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,10240,0.046879999339580536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,16384,0.06329599767923355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,65536,0.20931200683116913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,5120,0.03923200070858002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,3072,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,2560,0.02412799932062626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,3584,0.03126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,2048,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,4096,0.033055998384952545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,1536,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,1024,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,512,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,256,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,128,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,12288,0.07001599669456482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,65536,0.1921280026435852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,10240,0.06083200126886368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,8192,0.050175998359918594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,16384,0.0920960009098053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,7168,0.04467200115323067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,5120,0.033824000507593155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,4096,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,2560,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,2048,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,3072,0.026944000273942947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,3584,0.03145600110292435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,1536,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,512,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,128,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,256,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,12288,0.04374400153756142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,65536,0.14521600306034088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,8192,0.034623999148607254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,16384,0.05276799947023392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,7168,0.03174399957060814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,10240,0.040383998304605484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,5120,0.029888000339269638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,4096,0.03267199918627739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,3072,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,3584,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,2560,0.023615999147295952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,2048,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,1536,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,1024,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,512,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,12288,0.038943998515605927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,65536,0.17174400389194489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,10240,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,16384,0.047680001705884933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,8192,0.033440001308918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,7168,0.029952000826597214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,5120,0.023840000852942467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,3072,0.01775999926030636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,4096,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,3584,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,2048,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,1536,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,12288,0.03872000053524971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,7168,0.029920000582933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,8192,0.042399998754262924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,10240,0.05097600072622299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,16384,0.04451199993491173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,5120,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,65536,0.1088000014424324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,4096,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,3584,0.030592000111937523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,3072,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,2560,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,2048,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,512,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,1536,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,1024,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,256,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,128,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,12288,0.036288000643253326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,10240,0.03200000151991844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,65536,0.09836799651384354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,8192,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,7168,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,16384,0.04076800122857094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,5120,0.025472000241279602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,4096,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,2560,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,3584,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,3072,0.01744000054895878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,12288,0.03907199949026108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,8192,0.028063999488949776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,10240,0.03407999873161316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,7168,0.02735999971628189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,16384,0.04729599878191948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,5120,0.025280000641942024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,65536,0.08828800171613693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,3584,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,4096,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,3072,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,2560,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,2048,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,1536,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,512,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,65536,0.0936959981918335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,12288,0.03190400078892708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,8192,0.02768000029027462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,16384,0.036607999354600906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,10240,0.030079999938607216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,7168,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,4096,0.020864000543951988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,5120,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,3072,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,3584,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,512,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,12288,0.028063999488949776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,16384,0.03244800120592117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,10240,0.0331839993596077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,65536,0.07088000327348709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,7168,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,8192,0.029120000079274178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,5120,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,4096,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,3584,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,3072,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,2560,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,12288,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,16384,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,65536,0.06329599767923355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,8192,0.02316799946129322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,10240,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,7168,0.022207999601960182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,5120,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,4096,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,3584,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,3072,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,2048,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,12288,0.02470399998128414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,65536,0.053279999643564224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,7168,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,10240,0.02486399933695793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,8192,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,16384,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,5120,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,4096,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,3584,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,2048,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,1536,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,128,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,65536,0.047680001705884933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,12288,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,8192,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,7168,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,16384,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,10240,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,4096,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,5120,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,2048,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,3072,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,3584,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,1536,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,65536,0.04255999997258186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,5120,0.13462400436401367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,7168,0.18512000143527985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,12288,0.3056640028953552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,8192,0.20815999805927277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,10240,0.25894400477409363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,16384,0.40463998913764954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,4096,0.11075200140476227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,3584,0.09891200065612793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,3072,0.08742400258779526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,1536,0.05110400170087814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,2560,0.07500799745321274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,2048,0.06294400244951248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,1024,0.039103999733924866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,512,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,128,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,256,0.018239999189972878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,12288,0.122079998254776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,8192,0.08524800091981888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,7168,0.07599999755620956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,16384,0.1687680035829544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,10240,0.1074879989027977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,5120,0.056543998420238495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,4096,0.04979199916124344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,3072,0.03651199862360954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,3584,0.03094400092959404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,1024,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,2560,0.031647998839616776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,2048,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,1536,0.023264000192284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,256,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,128,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,512,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,12288,0.08339200168848038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,8192,0.0607680007815361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,10240,0.07177600264549255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,16384,0.10719999670982361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,7168,0.052960000932216644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,65536,0.7323520183563232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,5120,0.040192000567913055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,4096,0.033952001482248306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,3072,0.027744000777602196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,3584,0.03062400035560131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,2560,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,2048,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,1024,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,256,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,1536,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,128,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,512,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,12288,0.08371199667453766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,7168,0.05500800162553787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,8192,0.059039998799562454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,10240,0.07420799881219864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,16384,0.11177600175142288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,65536,0.3952000141143799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,5120,0.05119999870657921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,4096,0.03372799977660179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,3584,0.03094400092959404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,2560,0.029120000079274178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,3072,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,2048,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,1536,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,1024,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,512,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,256,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,12288,0.054976001381874084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,8192,0.04287999868392944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,10240,0.04944000020623207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,7168,0.04447999969124794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,16384,0.06691200286149979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,65536,0.41702398657798767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,5120,0.040063999593257904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,4096,0.03379200026392937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,3584,0.03046399913728237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,2560,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,3072,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,2048,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,1536,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,1024,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,512,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,256,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,12288,0.053408000618219376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,8192,0.041471999138593674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,10240,0.04527999833226204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,7168,0.04467200115323067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,65536,0.2152319997549057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,16384,0.06176000088453293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,5120,0.033952001482248306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,4096,0.033504001796245575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,3584,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,3072,0.02751999907195568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,2048,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,2560,0.024191999807953835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,1536,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,512,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,1024,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,128,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,256,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,12288,0.06892800331115723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,10240,0.03929600119590759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,65536,0.19142399728298187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,16384,0.09097599983215332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,8192,0.03683200106024742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,7168,0.045184001326560974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,5120,0.034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,4096,0.02844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,3072,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,2560,0.024032000452280045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,3584,0.030527999624609947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,2048,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,1536,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,512,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,256,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,1024,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,12288,0.04476799815893173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,8192,0.03670400008559227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,7168,0.03286400064826012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,65536,0.14560000598430634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,16384,0.05366399884223938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,10240,0.04028800129890442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,5120,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,3584,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,2560,0.024000000208616257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,2048,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,3072,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,4096,0.03292800113558769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,1536,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,1024,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,12288,0.04124800115823746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,65536,0.1728000044822693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,8192,0.035679999738931656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,10240,0.03705599904060364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,7168,0.030751999467611313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,16384,0.04499199986457825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,5120,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,4096,0.02070399932563305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,3072,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,3584,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,2048,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,1536,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,256,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,512,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,65536,0.10771200060844421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,12288,0.038015998899936676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,8192,0.03311999887228012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,16384,0.044224001467227936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,10240,0.0506879985332489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,7168,0.03964800015091896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,5120,0.029152000322937965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,4096,0.03276799991726875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,3072,0.02687999978661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,2560,0.023840000852942467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,3584,0.03001599945127964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,2048,0.020640000700950623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,1536,0.01788800023496151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,1024,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,512,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,256,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,65536,0.09689600020647049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,7168,0.028543999418616295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,12288,0.035551998764276505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,8192,0.0315839983522892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,10240,0.03200000151991844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,16384,0.040800001472234726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,5120,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,3072,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,4096,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,3584,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,256,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,12288,0.03388800099492073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,65536,0.09113600105047226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,7168,0.025760000571608543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,8192,0.027807999402284622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,10240,0.030559999868273735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,16384,0.03759999945759773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,5120,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,4096,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,2560,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,3584,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,3072,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,2048,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,1536,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,12288,0.03152000159025192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,65536,0.09504000097513199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,7168,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,8192,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,10240,0.0297279991209507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,16384,0.034432001411914825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,5120,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,4096,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,3072,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,2560,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,3584,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,12288,0.027168000116944313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,65536,0.070592001080513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,8192,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,10240,0.028063999488949776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,7168,0.024639999493956566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,16384,0.030112000182271004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,5120,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,4096,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,3584,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,2560,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,12288,0.024960000067949295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,65536,0.06537599861621857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,8192,0.023711999878287315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,10240,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,7168,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,16384,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,5120,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,3072,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,4096,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,1536,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,256,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,65536,0.04729599878191948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,12288,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,7168,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,10240,0.02579200081527233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,8192,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,16384,0.026976000517606735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,5120,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,4096,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,3072,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,3584,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,2048,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,65536,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,12288,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,16384,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,7168,0.02486399933695793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,8192,0.024960000067949295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,10240,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,4096,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,5120,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,3584,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,3072,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,1024,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,128,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,65536,0.0427200011909008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,5120,0.14390400052070618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,7168,0.19686399400234222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,8192,0.22739200294017792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,10240,0.27606400847435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,4096,0.1165120005607605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,16384,0.4355199933052063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,3072,0.088639996945858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,2560,0.07625599950551987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,2048,0.06454399973154068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,1536,0.05180799961090088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,3584,0.10252799838781357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,1024,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,128,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,12288,0.3263680040836334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,512,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,256,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,12288,0.13568000495433807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,5120,0.060896001756191254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,10240,0.11593600362539291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,8192,0.0928959995508194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,7168,0.08313599973917007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,16384,0.176256000995636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,4096,0.05363199859857559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,2560,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,3584,0.04793599992990494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,3072,0.0424639992415905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,2048,0.028384000062942505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,1536,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,128,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,1024,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,512,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,256,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,12288,0.08339200168848038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,10240,0.07126399874687195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,8192,0.05974400043487549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,7168,0.05580800026655197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,16384,0.10819199681282043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,65536,0.7551680207252502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,5120,0.04153599962592125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,3584,0.03152000159025192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,4096,0.033663999289274216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,3072,0.027871999889612198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,2560,0.02502400055527687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,2048,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,1536,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,512,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,1024,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,256,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,128,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,12288,0.08313599973917007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,8192,0.0613120011985302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,16384,0.11193600296974182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,10240,0.07472000271081924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,7168,0.0549440011382103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,5120,0.05161599814891815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,3584,0.030848000198602676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,3072,0.03519999980926514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,4096,0.04575999826192856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,2560,0.02486399933695793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,2048,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,1536,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,1024,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,512,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,256,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,65536,0.42659199237823486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,12288,0.0551999993622303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,65536,0.41648000478744507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,8192,0.04982399940490723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,10240,0.04870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,7168,0.04649600014090538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,16384,0.06646399945020676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,5120,0.04163200035691261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,4096,0.03619199991226196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,3584,0.032287999987602234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,3072,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,2560,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,2048,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,1536,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,1024,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,512,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,256,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,128,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,12288,0.05180799961090088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,10240,0.060256000608205795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,65536,0.23862400650978088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,8192,0.05011200159788132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,16384,0.062144000083208084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,7168,0.045152001082897186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,5120,0.033535998314619064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,3584,0.03174399957060814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,4096,0.03494400158524513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,3072,0.02768000029027462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,2560,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,1536,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,2048,0.017920000478625298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,512,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,1024,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,256,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,128,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,12288,0.07049600034952164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,65536,0.19120000302791595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,10240,0.05923200026154518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,16384,0.08975999802350998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,8192,0.049375999718904495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,7168,0.04540799930691719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,5120,0.03331200033426285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,4096,0.03363199904561043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,3584,0.024064000695943832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,3072,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,2560,0.023679999634623528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,2048,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,1536,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,1024,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,512,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,256,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,12288,0.045343998819589615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,10240,0.04118400067090988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,65536,0.15641599893569946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,7168,0.03385600075125694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,16384,0.0530879981815815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,8192,0.035551998764276505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,5120,0.025728000327944756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,3584,0.029920000582933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,3072,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,2560,0.02364799939095974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,2048,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,4096,0.03296000137925148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,1536,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,256,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,1024,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,12288,0.04047999903559685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,65536,0.16918399930000305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,8192,0.03510399907827377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,7168,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,10240,0.038656000047922134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,16384,0.045471999794244766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,5120,0.023679999634623528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,3584,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,4096,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,3072,0.01744000054895878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,1536,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,12288,0.03840000182390213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,65536,0.10940799862146378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,7168,0.03951999917626381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,8192,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,10240,0.034432001411914825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,16384,0.044863998889923096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,5120,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,4096,0.02505600079894066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,3072,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,2560,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,2048,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,3584,0.0297279991209507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,1536,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,1024,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,128,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,256,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,12288,0.03596799820661545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,65536,0.09616000205278397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,16384,0.04153599962592125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,8192,0.030112000182271004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,7168,0.028351999819278717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,10240,0.032255999743938446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,5120,0.02412799932062626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,4096,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,3584,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,2560,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,1536,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,128,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,12288,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,65536,0.09350399672985077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,16384,0.03868800029158592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,7168,0.02566399984061718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,8192,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,10240,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,5120,0.02191999927163124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,3584,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,4096,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,256,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,12288,0.03187200054526329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,65536,0.09561599791049957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,7168,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,8192,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,10240,0.028095999732613564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,16384,0.035711999982595444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,5120,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,4096,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,3584,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,2560,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,2048,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,1024,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,12288,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,65536,0.07043199986219406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,16384,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,10240,0.026240000501275063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,8192,0.024191999807953835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,7168,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,4096,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,5120,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,3584,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,2560,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,3072,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,2048,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,1536,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,12288,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,65536,0.0631679967045784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,10240,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,7168,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,8192,0.022752000018954277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,16384,0.02908799983561039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,5120,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,4096,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,3072,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,3584,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,2048,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,2560,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,1536,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,12288,0.028063999488949776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,65536,0.047488000243902206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,7168,0.026655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,8192,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,10240,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,16384,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,5120,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,4096,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,3584,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,1536,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,65536,0.04169600084424019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,12288,0.028704000636935234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,8192,0.027583999559283257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,10240,0.027583999559283257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,7168,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,16384,0.02796800062060356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,5120,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,4096,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,3584,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,3072,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,2560,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,2048,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,1536,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,65536,0.044544000178575516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,7168,0.14950400590896606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,5120,0.10921599715948105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,12288,0.24614399671554565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,8192,0.16921600699424744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,4096,0.09020800143480301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,10240,0.20787200331687927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,3584,0.08012799918651581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,2048,0.05209600180387497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,1536,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,1024,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,2560,0.06038400158286095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,3072,0.06992000341415405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,256,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,128,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,512,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,12288,0.0830719992518425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,16384,0.32524800300598145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,7168,0.052671998739242554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,8192,0.058720000088214874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,10240,0.07100799679756165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,16384,0.10732799768447876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,4096,0.03331200033426285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,5120,0.03948799893260002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,3584,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,3072,0.02723200060427189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,2560,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,2048,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,1536,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,512,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,256,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,1024,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,128,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,12288,0.06931199878454208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,7168,0.04560000076889992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,10240,0.06054399907588959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,8192,0.04950400069355965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,16384,0.09158399701118469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,5120,0.033984001725912094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,65536,0.3332799971103668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,4096,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,3584,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,2560,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,3072,0.02396799996495247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,2048,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,1536,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,1024,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,512,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,256,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,12288,0.07078400254249573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,16384,0.09100800007581711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,8192,0.05075199902057648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,10240,0.05936000123620033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,7168,0.045471999794244766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,65536,0.3322559893131256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,5120,0.033824000507593155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,3072,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,3584,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,4096,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,2560,0.02396799996495247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,2048,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,1536,0.018239999189972878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,512,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,256,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,1024,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,128,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,12288,0.04479999840259552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,8192,0.03651199862360954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,10240,0.04064000025391579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,65536,0.33481600880622864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,7168,0.032287999987602234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,16384,0.05558399856090546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,5120,0.028991999104619026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,3584,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,3072,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,4096,0.03280000016093254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,2048,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,2560,0.023744000121951103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,1536,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,1024,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,512,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,256,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,12288,0.043296001851558685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,7168,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,65536,0.17023999989032745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,8192,0.035360001027584076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,10240,0.03791999816894531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,16384,0.04975999891757965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,5120,0.02940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,2560,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,4096,0.03254399821162224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,3584,0.02956799976527691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,3072,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,2048,0.020640000700950623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,1536,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,1024,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,12288,0.059039998799562454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,7168,0.03852799907326698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,65536,0.1504639983177185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,8192,0.042399998754262924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,10240,0.05100800096988678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,16384,0.07548800110816956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,5120,0.02595200017094612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,4096,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,3584,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,2048,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,3072,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,1536,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,1024,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,512,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,256,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,128,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,12288,0.040608000010252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,65536,0.11238399893045425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,7168,0.027712000533938408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,10240,0.03683200106024742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,8192,0.03187200054526329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,16384,0.045951999723911285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,5120,0.023903999477624893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,4096,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,3584,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,3072,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,2560,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,2048,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,1536,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,512,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,128,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,12288,0.03235200047492981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,65536,0.09443199634552002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,8192,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,10240,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,16384,0.03750399872660637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,7168,0.026784000918269157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,5120,0.02271999977529049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,3584,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,4096,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,3072,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,2048,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,12288,0.03478400036692619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,7168,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,8192,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,10240,0.03155200183391571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,16384,0.03638400137424469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,65536,0.08665599673986435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,5120,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,4096,0.024064000695943832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,3584,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,2560,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,1536,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,12288,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,65536,0.07718399912118912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,7168,0.02470399998128414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,8192,0.026335999369621277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,10240,0.028960000723600388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,16384,0.03532800078392029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,5120,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,4096,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,3584,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,3072,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,2560,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,2048,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,1536,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,1024,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,12288,0.02800000086426735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,65536,0.07011199742555618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,8192,0.029632000252604485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,10240,0.03215999901294708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,7168,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,16384,0.03232000023126602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,5120,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,3072,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,3584,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,4096,0.02070399932563305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,2560,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,1536,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,256,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,12288,0.02783999964594841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,65536,0.08003199845552444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,8192,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,10240,0.02687999978661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,16384,0.03126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,7168,0.02412799932062626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,5120,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,4096,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,3584,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,2560,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,1024,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,12288,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,8192,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,7168,0.022304000332951546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,16384,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,65536,0.05523199960589409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,10240,0.024639999493956566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,5120,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,4096,0.018688000738620758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,3584,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,2048,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,65536,0.05337600037455559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,12288,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,7168,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,8192,0.02316799946129322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,16384,0.02687999978661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,10240,0.02412799932062626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,4096,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,5120,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,3584,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,2560,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,256,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,128,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,65536,0.04729599878191948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,12288,0.03174399957060814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,8192,0.02796800062060356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,10240,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,7168,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,16384,0.03232000023126602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,5120,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,4096,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,3584,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,2560,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,1536,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,65536,0.04224000126123428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,12288,0.030880000442266464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,7168,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,8192,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,10240,0.03161599859595299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,16384,0.031808000057935715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,5120,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,4096,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,3584,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,2560,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,1536,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,65536,0.04198399931192398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,5120,0.10950399935245514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,7168,0.14870400726795197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,12288,0.24553599953651428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,8192,0.1695680022239685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,10240,0.20768000185489655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,4096,0.09014400094747543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,2560,0.06035200133919716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,2048,0.05113599821925163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,16384,0.3259519934654236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,3584,0.08009599894285202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,3072,0.07052800059318542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,1536,0.04262400045990944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,1024,0.03219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,512,0.022431999444961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,128,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,256,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,12288,0.08294399827718735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,5120,0.04064000025391579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,10240,0.07107199728488922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,16384,0.09158399701118469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,8192,0.05926400050520897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,7168,0.04476799815893173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,4096,0.03308799862861633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,2560,0.02396799996495247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,3584,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,2048,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,3072,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,1536,0.017920000478625298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,1024,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,512,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,256,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,12288,0.06960000097751617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,8192,0.04915200173854828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,16384,0.09100800007581711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,7168,0.04444799944758415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,10240,0.05929600074887276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,5120,0.03747199848294258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,65536,0.3335359990596771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,4096,0.02879999950528145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,3072,0.023711999878287315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,3584,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,2560,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,2048,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,1024,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,1536,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,256,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,128,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,512,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,12288,0.07097599655389786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,8192,0.05027199909090996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,7168,0.04476799815893173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,10240,0.06111999973654747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,65536,0.32732799649238586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,16384,0.09139200299978256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,5120,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,4096,0.03324799984693527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,3072,0.026784000918269157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,3584,0.03001599945127964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,2048,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,2560,0.023679999634623528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,1536,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,1024,0.01375999953597784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,512,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,256,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,12288,0.04569600149989128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,7168,0.03254399821162224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,8192,0.03551999852061272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,10240,0.03996799886226654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,16384,0.05500800162553787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,5120,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,65536,0.32678401470184326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,3584,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,4096,0.0326399989426136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,3072,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,2560,0.023679999634623528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,2048,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,1536,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,1024,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,128,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,12288,0.04297599941492081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,8192,0.03510399907827377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,7168,0.0331839993596077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,65536,0.17023999989032745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,10240,0.03855999931693077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,16384,0.05129599943757057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,5120,0.02985600009560585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,3584,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,4096,0.03280000016093254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,3072,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,2560,0.02377600036561489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,2048,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,1536,0.01775999926030636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,512,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,1024,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,256,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,12288,0.05923200026154518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,8192,0.04230400174856186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,10240,0.050624001771211624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,65536,0.15065599977970123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,16384,0.0753600001335144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,7168,0.03807999938726425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,5120,0.029823999851942062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,2560,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,3072,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,4096,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,3584,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,2048,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,1536,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,512,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,1024,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,256,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,65536,0.11260800063610077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,12288,0.04028800129890442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,7168,0.031968001276254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,8192,0.03171199932694435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,10240,0.03702399879693985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,16384,0.046751998364925385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,4096,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,2560,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,5120,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,3584,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,2048,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,1536,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,12288,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,65536,0.0960640013217926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,7168,0.02924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,8192,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,10240,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,16384,0.03622400015592575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,5120,0.023296000435948372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,4096,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,2048,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,3072,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,65536,0.08687999844551086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,12288,0.03478400036692619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,7168,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,8192,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,10240,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,16384,0.03657599911093712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,5120,0.027936000376939774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,4096,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,3072,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,3584,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,2048,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,1536,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,512,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,12288,0.03155200183391571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,65536,0.07731200009584427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,7168,0.02457600086927414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,8192,0.025599999353289604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,10240,0.02800000086426735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,16384,0.035360001027584076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,5120,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,4096,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,3584,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,2560,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,1536,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,12288,0.031968001276254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,65536,0.07004799693822861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,7168,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,8192,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,10240,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,16384,0.03267199918627739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,5120,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,2560,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,3584,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,4096,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,1024,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,65536,0.08057600259780884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,12288,0.027615999802947044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,8192,0.023615999147295952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,7168,0.02396799996495247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,10240,0.026176000013947487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,16384,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,4096,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,5120,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,3584,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,2560,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,2048,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,3072,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,1536,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,128,0.008128000423312187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,65536,0.05516799911856651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,12288,0.0261439997702837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,16384,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,7168,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,8192,0.022304000332951546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,10240,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,4096,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,5120,0.020896000787615776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,3072,0.01692800037562847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,3584,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,2560,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,2048,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,512,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,12288,0.025919999927282333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,65536,0.053408000618219376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,8192,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,16384,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,10240,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,7168,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,5120,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,4096,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,2048,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,12288,0.03433600068092346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,65536,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,16384,0.03564799949526787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,8192,0.02800000086426735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,10240,0.03347200155258179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,7168,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,4096,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,5120,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,3584,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,3072,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,1024,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,1536,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,65536,0.0427200011909008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,12288,0.03455999866127968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,8192,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,10240,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,7168,0.025887999683618546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,16384,0.03488000109791756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,5120,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,4096,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,3584,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,2560,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,1024,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,1536,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,128,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,65536,0.04297599941492081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,5120,0.09427200257778168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,7168,0.12732799351215363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,8192,0.14396800100803375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,12288,0.21107199788093567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,10240,0.17788800597190857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,4096,0.07689599692821503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,16384,0.276095986366272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,3584,0.06880000233650208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,2560,0.05190400034189224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,3072,0.06080000102519989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,2048,0.04399999976158142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,1536,0.03590400144457817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,1024,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,256,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,512,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,128,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,12288,0.05967999994754791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,8192,0.05756799876689911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,7168,0.05209600180387497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,5120,0.04028800129890442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,10240,0.0695360004901886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,16384,0.104032002389431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,4096,0.03280000016093254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,3584,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,3072,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,2560,0.023615999147295952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,2048,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,1536,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,512,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,1024,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,256,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,12288,0.05907199904322624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,8192,0.04287999868392944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,7168,0.03868800029158592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,10240,0.051072001457214355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,16384,0.07526399940252304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,3072,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,3584,0.023104000836610794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,5120,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,65536,0.38915199041366577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,4096,0.02502400055527687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,2560,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,2048,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,1024,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,1536,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,512,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,256,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,128,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,12288,0.061792001128196716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,7168,0.040031999349594116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,8192,0.044096000492572784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,10240,0.052928000688552856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,65536,0.27401599287986755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,16384,0.07827199995517731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,5120,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,3584,0.029632000252604485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,3072,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,4096,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,2560,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,2048,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,1536,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,1024,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,512,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,256,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,128,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,12288,0.0424639992415905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,10240,0.03840000182390213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,8192,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,16384,0.050016000866889954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,7168,0.03299200162291527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,65536,0.28748801350593567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,5120,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,4096,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,3584,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,3072,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,2048,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,12288,0.040608000010252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,65536,0.14934399724006653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,8192,0.031808000057935715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,10240,0.036607999354600906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,7168,0.02864000014960766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,16384,0.04652800038456917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,5120,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,4096,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,2560,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,3072,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,512,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,12288,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,7168,0.035392001271247864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,8192,0.03932800143957138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,10240,0.030079999938607216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,16384,0.036768000572919846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,65536,0.1361279934644699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,5120,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,3584,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,3072,0.01692800037562847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,4096,0.023679999634623528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,2048,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,12288,0.03862399980425835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,65536,0.09852799773216248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,7168,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,8192,0.0315839983522892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,16384,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,10240,0.03500799834728241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,5120,0.024032000452280045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,4096,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,3072,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,2048,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,3584,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,12288,0.028736000880599022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,65536,0.08256000280380249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,7168,0.023615999147295952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,8192,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,10240,0.0272000003606081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,16384,0.03174399957060814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,5120,0.022207999601960182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,4096,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,3584,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,2560,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,2048,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,128,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,12288,0.03331200033426285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,8192,0.026048000901937485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,65536,0.07606399804353714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,10240,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,7168,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,16384,0.03558399900794029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,5120,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,4096,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,3584,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,3072,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,1536,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,512,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,12288,0.02796800062060356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,65536,0.06774400174617767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,7168,0.023072000592947006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,8192,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,16384,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,10240,0.025887999683618546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,4096,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,5120,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,2560,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,3072,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,3584,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,1536,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,1024,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,12288,0.026464000344276428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,65536,0.060416001826524734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,7168,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,8192,0.02316799946129322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,10240,0.025151999667286873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,16384,0.030400000512599945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,5120,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,4096,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,3584,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,3072,0.017983999103307724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,2048,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,12288,0.026048000901937485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,65536,0.07340800017118454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,10240,0.024639999493956566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,8192,0.022655999287962914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,7168,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,16384,0.02783999964594841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,5120,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,4096,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,3584,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,2560,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,1536,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,12288,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,65536,0.047648001462221146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,8192,0.023231999948620796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,7168,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,10240,0.024320000782608986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,16384,0.02582399919629097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,4096,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,5120,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,2560,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,3584,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,12288,0.030592000111937523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,65536,0.04879999905824661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,8192,0.02844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,7168,0.026048000901937485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,10240,0.030112000182271004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,16384,0.030751999467611313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,5120,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,4096,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,3584,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,2048,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,1536,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,65536,0.04102399945259094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,12288,0.034304000437259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,10240,0.030719999223947525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,7168,0.02566399984061718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,8192,0.02691200003027916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,16384,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,5120,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,4096,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,3584,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,2560,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,2048,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,1536,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,256,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,12288,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,65536,0.0488319993019104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,8192,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,10240,0.029952000826597214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,16384,0.03929600119590759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,7168,0.024447999894618988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,5120,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,4096,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,3072,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,3584,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,2560,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,1536,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,256,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,65536,0.05008000135421753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,12288,0.1905599981546402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,5120,0.08640000224113464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,8192,0.1313920021057129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,7168,0.11638399958610535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,10240,0.1621440052986145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,4096,0.07177600264549255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,16384,0.2486400008201599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,3072,0.056832000613212585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,3584,0.06387200206518173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,2560,0.04851200059056282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,2048,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,1536,0.033695999532938004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,1024,0.026079999282956123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,256,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,128,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,512,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,12288,0.07763200253248215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,10240,0.06703999638557434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,7168,0.0504320003092289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,5120,0.0387520007789135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,8192,0.055904000997543335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,16384,0.10051199793815613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,4096,0.0318400003015995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,3584,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,3072,0.026655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,1536,0.017503999173641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,2048,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,2560,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,1024,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,256,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,512,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,12288,0.054048001766204834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,10240,0.04681599885225296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,7168,0.03468799963593483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,8192,0.03961599990725517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,16384,0.06841599941253662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,5120,0.028095999732613564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,65536,0.3660160005092621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,4096,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,3584,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,3072,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,2560,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,2048,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,1024,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,512,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,1536,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,128,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,12288,0.05500800162553787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,8192,0.03932800143957138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,10240,0.04684799909591675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,7168,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,65536,0.24646399915218353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,16384,0.0695360004901886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,5120,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,4096,0.02348800003528595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,3584,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,2048,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,3072,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,2560,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,1536,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,1024,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,256,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,512,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,128,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,12288,0.044064000248909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,10240,0.03747199848294258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,8192,0.033215999603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,7168,0.03017600066959858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,16384,0.04755200073122978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,65536,0.2508479952812195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,5120,0.025760000571608543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,4096,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,3072,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,3584,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,1536,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,512,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,12288,0.03903999924659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,65536,0.1380160003900528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,8192,0.02985600009560585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,10240,0.035232000052928925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,16384,0.04499199986457825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,7168,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,5120,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,3584,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,4096,0.020800000056624413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,3072,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,2048,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,1536,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,1024,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,12288,0.032575998455286026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,7168,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,8192,0.02611199952661991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,10240,0.02687999978661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,65536,0.12163200229406357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,16384,0.03798399865627289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,5120,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,3584,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,4096,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,3072,0.01692800037562847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,2048,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,256,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,128,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,12288,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,65536,0.09267199784517288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,8192,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,7168,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,10240,0.026976000517606735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,16384,0.03161599859595299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,5120,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,2560,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,4096,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,3584,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,2048,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,1536,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,128,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,12288,0.02687999978661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,65536,0.07702399790287018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,8192,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,10240,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,7168,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,16384,0.030848000198602676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,5120,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,4096,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,3584,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,3072,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,1024,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,128,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,12288,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,8192,0.023264000192284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,10240,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,65536,0.06921599805355072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,7168,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,16384,0.02844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,5120,0.020896000787615776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,4096,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,3584,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,3072,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,2048,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,2560,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,512,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,256,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,12288,0.02691200003027916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,65536,0.06275200098752975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,8192,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,10240,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,16384,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,7168,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,5120,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,4096,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,3584,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,3072,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,1536,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,128,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,12288,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,65536,0.0551999993622303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,8192,0.02287999913096428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,16384,0.02796800062060356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,10240,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,7168,0.022304000332951546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,5120,0.022495999932289124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,4096,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,3584,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,3072,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,2048,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,1536,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,12288,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,65536,0.049056001007556915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,8192,0.02502400055527687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,16384,0.02751999907195568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,10240,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,7168,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,5120,0.022752000018954277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,4096,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,2560,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,3584,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,3072,0.01926399953663349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,2048,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,128,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,65536,0.043168000876903534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,12288,0.0318400003015995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,8192,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,7168,0.025599999353289604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,16384,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,10240,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,5120,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,4096,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,3584,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,2048,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,3072,0.01775999926030636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,1536,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,12288,0.034591998904943466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,65536,0.0427200011909008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,7168,0.02582399919629097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,8192,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,10240,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,16384,0.04195199906826019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,5120,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,4096,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,3584,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,3072,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,2048,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,1024,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,128,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,65536,0.04854400083422661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,12288,0.03388800099492073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,10240,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,7168,0.025919999927282333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,8192,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,16384,0.041919998824596405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,4096,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,5120,0.02070399932563305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,3072,0.017503999173641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,3584,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,2560,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,2048,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,65536,0.0756480023264885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,12288,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,8192,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,10240,0.029952000826597214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,16384,0.03977600112557411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,7168,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,5120,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,4096,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,3584,0.018079999834299088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,3072,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,1536,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,2048,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,65536,0.07548800110816956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,5120,0.08659200370311737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,4096,0.07062400132417679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,7168,0.11555200070142746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,8192,0.13094399869441986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,12288,0.19062399864196777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,10240,0.16089600324630737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,1536,0.03340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,3584,0.0629120022058487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,2048,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,2560,0.047775998711586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,3072,0.05548800155520439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,16384,0.24953599274158478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,1024,0.02595200017094612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,128,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,512,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,256,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,12288,0.05673599988222122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,10240,0.04806400090456009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,8192,0.0387520007789135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,7168,0.03542400151491165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,5120,0.029184000566601753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,16384,0.0684799998998642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,4096,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,3584,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,3072,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,2560,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,1536,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,2048,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,1024,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,256,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,512,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,12288,0.0544000007212162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,10240,0.046720001846551895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,8192,0.03929600119590759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,16384,0.06844799965620041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,7168,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,5120,0.028384000062942505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,4096,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,3584,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,2560,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,3072,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,65536,0.25011199712753296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,2048,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,1536,0.014495999552309513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,1024,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,256,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,512,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,12288,0.054207999259233475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,7168,0.03468799963593483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,8192,0.03929600119590759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,10240,0.061184000223875046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,16384,0.06812799721956253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,65536,0.24643200635910034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,5120,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,3584,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,2560,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,3072,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,4096,0.023711999878287315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,2048,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,1536,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,512,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,128,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,256,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,12288,0.041152000427246094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,10240,0.037408001720905304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,16384,0.04742399975657463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,7168,0.029983999207615852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,8192,0.033215999603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,65536,0.246848002076149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,5120,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,4096,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,3584,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,3072,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,2048,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,1536,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,1024,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,256,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,12288,0.038336001336574554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,10240,0.03411199897527695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,8192,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,65536,0.13465599715709686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,16384,0.04355200007557869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,7168,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,5120,0.031968001276254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,4096,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,3584,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,1536,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,1024,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,12288,0.031488001346588135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,16384,0.03686400130391121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,8192,0.025151999667286873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,7168,0.02332800067961216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,10240,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,65536,0.12137600034475327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,5120,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,4096,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,2560,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,3072,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,3584,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,2048,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,1536,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,12288,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,7168,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,8192,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,10240,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,16384,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,65536,0.09328000247478485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,5120,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,3584,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,4096,0.017503999173641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,1024,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,12288,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,7168,0.023135999217629433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,8192,0.023264000192284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,65536,0.07705599814653397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,16384,0.029503999277949333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,10240,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,5120,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,4096,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,3584,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,3072,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,1024,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,12288,0.02985600009560585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,65536,0.06963200122117996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,7168,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,8192,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,10240,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,16384,0.03606399893760681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,5120,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,2560,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,3584,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,3072,0.014495999552309513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,4096,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,1536,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,1024,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,256,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,512,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,65536,0.06278400123119354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,12288,0.02579200081527233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,8192,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,10240,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,7168,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,16384,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,4096,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,5120,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,3072,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,3584,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,2560,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,2048,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,1536,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,65536,0.05526399984955788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,12288,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,16384,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,10240,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,8192,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,7168,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,5120,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,4096,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,2560,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,3072,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,3584,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,2048,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,1536,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,256,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,1024,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,128,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,12288,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,65536,0.04931199923157692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,7168,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,16384,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,8192,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,10240,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,5120,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,4096,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,3584,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,3072,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,2560,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,2048,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,1024,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,256,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,1536,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,65536,0.04419200122356415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,12288,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,8192,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,10240,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,7168,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,16384,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,4096,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,5120,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,3584,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,3072,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,2560,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,2048,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,1536,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,1024,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,12288,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,65536,0.048608001321554184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,8192,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,7168,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,16384,0.017503999173641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,10240,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,5120,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,3584,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,3072,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,4096,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,2048,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,2560,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,1536,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,1024,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,65536,0.04531199857592583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,12288,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,16384,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,8192,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,10240,0.01375999953597784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,7168,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,4096,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,5120,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,3584,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,3072,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,2048,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,2560,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,1536,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,1024,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,512,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,256,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,128,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,65536,0.04476799815893173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,12288,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,10240,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,8192,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,16384,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,5120,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,7168,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,4096,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,3584,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,3072,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,2560,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,2048,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,1024,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,1536,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,512,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,65536,0.0453759990632534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,5120,0.08611200004816055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,7168,0.11497599631547928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,12288,0.19094400107860565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,4096,0.07215999811887741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,8192,0.13171200454235077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,10240,0.16102400422096252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,3584,0.062463998794555664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,16384,0.25174400210380554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,2560,0.04879999905824661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,1536,0.033952001482248306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,2048,0.040383998304605484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,3072,0.05536000058054924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,1024,0.02582399919629097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,512,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,256,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,128,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,12288,0.0544000007212162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,8192,0.04527999833226204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,10240,0.04822399839758873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,7168,0.036639999598264694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,5120,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,16384,0.06995200365781784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,4096,0.024000000208616257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,3584,0.022336000576615334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,2560,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,3072,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,2048,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,1536,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,1024,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,512,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,256,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,128,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,12288,0.04944000020623207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,8192,0.03561599925160408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,10240,0.044064000248909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,16384,0.061216000467538834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,7168,0.03356799855828285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,5120,0.028095999732613564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,4096,0.023840000852942467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,3584,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,2560,0.018239999189972878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,3072,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,65536,0.24899199604988098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,2048,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,1536,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,1024,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,256,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,512,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,12288,0.04214400053024292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,16384,0.053247999399900436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,7168,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,10240,0.03840000182390213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,8192,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,65536,0.2046079933643341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,5120,0.025728000327944756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,4096,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,3584,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,3072,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,2560,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,2048,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,1536,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,1024,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,512,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,256,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,128,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,12288,0.03481600061058998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,8192,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,10240,0.03254399821162224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,16384,0.04364800080657005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,7168,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,5120,0.022975999861955643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,65536,0.1736000031232834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,3584,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,4096,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,3072,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,2048,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,1536,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,2560,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,1024,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,512,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,128,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,12288,0.03248000144958496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,8192,0.024607999250292778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,10240,0.029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,7168,0.02316799946129322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,16384,0.04044799879193306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,65536,0.1327040046453476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,5120,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,4096,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,3072,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,2048,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,3584,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,2560,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,1536,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,1024,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,512,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,128,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,12288,0.025119999423623085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,7168,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,8192,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,10240,0.023584000766277313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,65536,0.12720000743865967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,16384,0.030912000685930252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,5120,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,4096,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,3584,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,3072,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,2560,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,2048,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,1536,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,12288,0.025151999667286873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,7168,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,8192,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,10240,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,16384,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,65536,0.09071999788284302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,5120,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,4096,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,2560,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,3072,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,3584,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,2048,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,1536,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,256,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,65536,0.0767040029168129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,12288,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,7168,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,8192,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,10240,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,16384,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,5120,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,4096,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,2048,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,2560,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,3072,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,3584,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,1024,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,512,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,12288,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,65536,0.06995200365781784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,8192,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,7168,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,10240,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,16384,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,5120,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,3072,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,3584,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,2048,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,4096,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,2560,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,512,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,1536,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,256,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,1024,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,12288,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,65536,0.060447998344898224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,8192,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,10240,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,7168,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,16384,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,4096,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,5120,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,3584,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,3072,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,2048,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,2560,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,1024,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,1536,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,256,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,512,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,65536,0.0549440011382103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,12288,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,8192,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,10240,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,7168,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,16384,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,5120,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,4096,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,3584,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,3072,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,2560,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,2048,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,1536,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,1024,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,512,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,256,0.007968000136315823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,12288,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,65536,0.04646399989724159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,7168,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,8192,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,10240,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,16384,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,5120,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,4096,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,2048,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,3584,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,2560,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,3072,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,1536,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,128,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,256,0.008128000423312187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,512,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,1024,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,65536,0.04444799944758415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,12288,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,8192,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,10240,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,7168,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,16384,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,4096,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,5120,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,3072,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,3584,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,2560,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,2048,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,1536,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,1024,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,256,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,512,0.007968000136315823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,128,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,65536,0.03811199963092804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,12288,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,8192,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,10240,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,16384,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,7168,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,5120,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,4096,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,2560,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,3584,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,3072,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,1024,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,1536,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,2048,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,128,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,256,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,512,0.007968000136315823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,12288,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,65536,0.03551999852061272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,7168,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,8192,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,10240,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,16384,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,5120,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,4096,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,3584,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,3072,0.007968000136315823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,2048,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,2560,0.007968000136315823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,1024,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,1536,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,512,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,256,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,128,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,12288,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,65536,0.035679999738931656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,8192,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,10240,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,7168,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,16384,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,4096,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,5120,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,3584,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,3072,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,2560,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,2048,0.007968000136315823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,1536,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,512,0.007968000136315823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,1024,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,256,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,128,0.007424000184983015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,65536,0.0352960005402565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,5120,0.07772800326347351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,4096,0.05804799869656563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,7168,0.09244800359010696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,8192,0.10044799745082855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,12288,0.14431999623775482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,10240,0.12703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,3584,0.05584000051021576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,3072,0.05056000128388405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,2560,0.047488000243902206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,16384,0.18611200153827667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,1536,0.03296000137925148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,1024,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,2048,0.04105599969625473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,256,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,128,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,512,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,12288,0.04294399917125702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,5120,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,10240,0.03932800143957138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,8192,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,7168,0.02985600009560585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,16384,0.054687999188899994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,4096,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,3584,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,2560,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,2048,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,3072,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,1536,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,1024,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,512,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,256,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,128,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,12288,0.03481600061058998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,8192,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,7168,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,10240,0.03049599938094616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,16384,0.0427200011909008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,4096,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,5120,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,3584,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,3072,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,2560,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,65536,0.18454399704933167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,1536,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,2048,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,1024,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,512,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,256,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,12288,0.03110400028526783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,8192,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,10240,0.027871999889612198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,7168,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,16384,0.03846399858593941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,65536,0.14083200693130493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,5120,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,4096,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,3584,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,3072,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,2560,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,2048,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,512,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,128,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,256,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,12288,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,10240,0.022655999287962914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,8192,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,7168,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,16384,0.030912000685930252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,5120,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,65536,0.12678399682044983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,4096,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,3584,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,3072,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,2560,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,2048,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,1536,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,1024,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,12288,0.02457600086927414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,8192,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,7168,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,10240,0.022655999287962914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,16384,0.029791999608278275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,65536,0.09750399738550186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,5120,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,4096,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,3072,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,3584,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,2560,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,2048,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,1536,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,256,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,512,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,1024,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,128,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,12288,0.022048000246286392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,7168,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,8192,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,10240,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,16384,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,65536,0.0931520015001297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,5120,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,4096,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,2560,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,3072,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,3584,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,2048,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,1536,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,512,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,256,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,128,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,1024,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,12288,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,7168,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,8192,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,10240,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,16384,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,65536,0.08204799890518188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,5120,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,4096,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,3584,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,3072,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,2560,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,2048,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,1024,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,512,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,1536,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,256,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,128,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,12288,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,65536,0.052928000688552856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,7168,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,10240,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,8192,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,16384,0.01881599985063076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,5120,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,4096,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,3584,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,3072,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,2560,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,2048,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,1536,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,1024,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,256,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,512,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,128,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,12288,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,7168,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,8192,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,10240,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,65536,0.051392000168561935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,16384,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,5120,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,3072,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,4096,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,3584,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,2560,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,2048,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,1536,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,256,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,1024,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,512,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,128,0.007040000054985285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,65536,0.04358400031924248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,12288,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,8192,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,10240,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,16384,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,7168,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,5120,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,4096,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,3584,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,3072,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,2560,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,2048,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,1536,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,1024,0.007552000228315592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,256,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,512,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,128,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,12288,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,65536,0.04182400181889534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,16384,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,8192,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,10240,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,7168,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,5120,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,4096,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,2560,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,3072,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,2048,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,3584,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,1536,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,1024,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,256,0.007071999832987785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,512,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,128,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,12288,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,65536,0.03593600168824196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,8192,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,7168,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,10240,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,16384,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,5120,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,3072,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,3584,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,4096,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,2048,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,2560,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,1536,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,128,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,512,0.007424000184983015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,1024,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,256,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,65536,0.033344000577926636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,12288,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,8192,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,16384,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,10240,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,7168,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,5120,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,4096,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,3584,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,2560,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,3072,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,2048,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,1536,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,1024,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,256,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,512,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,128,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,65536,0.02956799976527691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,12288,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,16384,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,8192,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,7168,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,10240,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,4096,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,5120,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,3584,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,2048,0.007424000184983015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,2560,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,3072,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,1024,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,1536,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,512,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,256,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,128,0.0072639998979866505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,12288,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,65536,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,10240,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,7168,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,8192,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,16384,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,5120,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,4096,0.007424000184983015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,3072,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,2560,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,3584,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,2048,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,1536,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,1024,0.0072639998979866505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,128,0.007040000054985285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,256,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,512,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,65536,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,12288,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,7168,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,8192,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,10240,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,16384,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,5120,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,4096,0.007424000184983015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,3072,0.0074880002066493034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,2560,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,3584,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,2048,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,1536,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,512,0.007391999941319227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,256,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,128,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,1024,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,65536,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,5120,0.06883200258016586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,4096,0.05167999863624573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,7168,0.08444800227880478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,8192,0.09324800223112106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,12288,0.13596799969673157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,10240,0.11785600334405899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,16384,0.1775680035352707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,1536,0.0331839993596077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,3584,0.047359999269247055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,2048,0.03519999980926514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,2560,0.039264000952243805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,3072,0.04224000126123428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,1024,0.029888000339269638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,512,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,256,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,128,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,12288,0.040863998234272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,10240,0.035071998834609985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,8192,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,7168,0.026335999369621277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,5120,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,16384,0.051231998950242996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,4096,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,3584,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,2560,0.015039999969303608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,3072,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,2048,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,1536,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,1024,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,512,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,256,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,128,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,12288,0.034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,10240,0.030527999624609947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,7168,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,8192,0.02470399998128414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,16384,0.04265600070357323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,5120,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,4096,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,3584,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,3072,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,65536,0.17923200130462646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,2560,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,2048,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,1024,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,512,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,256,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,128,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,12288,0.03110400028526783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,8192,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,7168,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,10240,0.02816000021994114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,16384,0.04025600105524063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,4096,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,5120,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,65536,0.14812800288200378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,3584,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,3072,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,2560,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,1536,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,512,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,256,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,2048,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,128,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,12288,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,10240,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,8192,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,7168,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,16384,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,65536,0.13251200318336487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,5120,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,4096,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,3072,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,3584,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,2560,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,2048,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,1536,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,1024,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,512,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,12288,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,7168,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,8192,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,10240,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,16384,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,65536,0.09487999975681305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,5120,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,3072,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,2560,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,3584,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,4096,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,2048,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,1536,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,1024,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,512,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,12288,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,16384,0.022207999601960182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,8192,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,10240,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,7168,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,65536,0.08473599702119827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,5120,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,4096,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,3584,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,3072,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,2048,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,2560,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,1536,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,1024,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,256,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,512,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,12288,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,7168,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,65536,0.06627199798822403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,8192,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,10240,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,16384,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,5120,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,3584,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,4096,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,2560,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,3072,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,2048,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,1536,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,512,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,256,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,1024,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,12288,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,65536,0.05833600088953972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,8192,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,7168,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,10240,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,16384,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,5120,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,3584,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,4096,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,2560,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,3072,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,2048,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,1536,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,256,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,1024,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,512,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,12288,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,8192,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,10240,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,65536,0.05446400120854378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,7168,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,16384,0.018688000738620758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,5120,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,4096,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,3072,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,2048,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,3584,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,2560,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,1536,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,1024,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,256,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,512,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,65536,0.05104000121355057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,12288,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,7168,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,10240,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,8192,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,16384,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,5120,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,4096,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,3584,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,3072,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,2560,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,2048,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,1536,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,1024,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,256,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,512,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,128,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,12288,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,65536,0.04956800118088722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,16384,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,8192,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,7168,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,10240,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,5120,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,4096,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,3584,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,2560,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,3072,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,2048,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,1536,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,512,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,1024,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,256,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,128,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,12288,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,65536,0.04560000076889992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,8192,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,16384,0.01692800037562847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,7168,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,10240,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,4096,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,5120,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,3072,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,3584,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,2560,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,2048,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,1024,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,1536,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,256,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,512,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,128,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,65536,0.04339199885725975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,12288,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,7168,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,10240,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,8192,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,16384,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,5120,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,4096,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,3584,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,2560,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,3072,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,2048,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,1536,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,1024,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,256,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,128,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,512,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,12288,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,65536,0.04179200157523155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,16384,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,7168,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,8192,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,10240,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,5120,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,3584,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,2560,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,3072,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,4096,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,2048,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,1536,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,512,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,256,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,1024,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,128,0.008128000423312187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,65536,0.04150399938225746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,12288,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,8192,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,16384,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,7168,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,10240,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,5120,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,4096,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,3584,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,3072,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,2048,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,2560,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,1536,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,1024,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,512,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,256,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,128,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,65536,0.04076800122857094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,12288,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,16384,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,7168,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,8192,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,10240,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,5120,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,3584,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,4096,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,3072,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,2048,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,2560,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,1536,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,1024,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,512,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,128,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,256,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,65536,0.04092799872159958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,4096,4.528448104858398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,5120,5.524320125579834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,7168,7.497600078582764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,3584,4.028672218322754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,8192,8.474495887756348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,3072,3.516063928604126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,1024,1.5390080213546753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,2560,3.0142719745635986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,1536,2.0332159996032715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,2048,2.5172159671783447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,10240,10.453856468200684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,512,1.048192024230957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,256,0.7636479735374451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,128,0.5905280113220215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,12288,12.42416000366211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,8192,2.1731200218200684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,10240,2.6801600456237793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,12288,3.184767961502075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,7168,1.923807978630066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,5120,1.4180799722671509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,4096,1.1643199920654297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,16384,4.1927361488342285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,3584,1.0459519624710083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,3072,0.9101759791374207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,2048,0.6515520215034485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,2560,0.7803840041160583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,1024,0.3975040018558502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,512,0.27353599667549133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,1536,0.5244799852371216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,128,0.15334400534629822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,256,0.19760000705718994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,16384,16.362272262573242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,8192,1.632256031036377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,7168,1.4448959827423096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,10240,2.0127038955688477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,12288,2.391263961791992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,5120,1.0665600299835205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,4096,0.875648021697998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,3584,0.783456027507782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,3072,0.6886079907417297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,16384,3.1469759941101074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,2048,0.4954240024089813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,2560,0.5902400016784668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,1536,0.4007680118083954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,1024,0.3054080009460449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,256,0.15039999783039093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,512,0.20854400098323822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,128,0.11776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,8192,1.3614399433135986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,10240,1.6781760454177856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,12288,1.9939520359039307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,5120,0.8895360231399536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,7168,1.2051520347595215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,16384,2.6233599185943604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,4096,0.7305600047111511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,2048,0.4143039882183075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,2560,0.49459201097488403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,3072,0.5749120116233826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,3584,0.6538879871368408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,512,0.17552000284194946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,1024,0.256415992975235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,256,0.12697599828243256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,1536,0.33555200695991516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,128,0.09935999661684036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,10240,1.3432960510253906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,12288,1.5957119464874268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,16384,2.0991361141204834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,8192,1.0897279977798462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,7168,0.964896023273468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,5120,0.712448000907898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,4096,0.5854719877243042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,3584,0.5238720178604126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,3072,0.4610559940338135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,2048,0.3322240114212036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,2560,0.39660799503326416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,1536,0.26927998661994934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,512,0.14003199338912964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,1024,0.20473599433898926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,256,0.10252799838781357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,128,0.08070400357246399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,65536,18.94620704650879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,12288,1.3979519605636597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,16384,1.8386880159378052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,65536,13.820256233215332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,8192,0.9553920030593872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,10240,1.1773120164871216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,7168,0.8461120128631592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,5120,0.6247040033340454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,4096,0.514240026473999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,3584,0.45977601408958435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,3072,0.404448002576828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,2560,0.34883201122283936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,2048,0.2926720082759857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,1536,0.2362239956855774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,512,0.12387199699878693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,1024,0.1799039989709854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,256,0.09100800007581711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,128,0.07171200215816498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,65536,11.569215774536133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,8192,0.6840959787368774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,10240,0.8430399894714355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,12288,1.0007679462432861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,16384,1.315168023109436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,5120,0.4484800100326538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,4096,0.36902400851249695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,7168,0.6066880226135254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,65536,8.965503692626953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,2560,0.2512960135936737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,1536,0.17177599668502808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,3072,0.2911359965801239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,2048,0.2109760046005249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,3584,0.33052799105644226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,256,0.06748799979686737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,128,0.05382400006055832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,512,0.09097599983215332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,1024,0.1316159963607788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,8192,0.5477120280265808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,10240,0.6752640008926392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,12288,0.8011199831962585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,7168,0.48601600527763367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,5120,0.3596479892730713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,16384,1.0548800230026245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,4096,0.2963840067386627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,2560,0.20192000269889832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,2048,0.16918399930000305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,3072,0.23350399732589722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,3584,0.26531198620796204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,512,0.07327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,256,0.056095998734235764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,1024,0.10569600015878677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,1536,0.1377280056476593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,128,0.04492799937725067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,65536,8.026752471923828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,10240,0.5922240018844604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,12288,0.7025279998779297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,16384,0.9223999977111816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,8192,0.4808320105075836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,5120,0.315744012594223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,7168,0.42688000202178955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,3584,0.23369599878787994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,4096,0.2603519856929779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,3072,0.20640000700950623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,2560,0.17798399925231934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,1024,0.09385599941015244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,1536,0.1215360015630722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,2048,0.14950400590896606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,512,0.06508799642324448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,128,0.040031999349594116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,256,0.0498879998922348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,65536,5.816448211669922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,10240,0.5090879797935486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,12288,0.6038720011711121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,16384,0.791808009147644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,8192,0.4131520092487335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,5120,0.2726080119609833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,7168,0.3668160140514374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,3584,0.20076799392700195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,4096,0.22390399873256683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,2560,0.15331199765205383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,3072,0.17766399681568146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,1536,0.10505600273609161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,2048,0.12908799946308136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,512,0.05651199817657471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,1024,0.08124800026416779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,65536,4.787712097167969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,256,0.04335999861359596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,128,0.03551999852061272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,8192,0.3461439907550812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,10240,0.4259839951992035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,12288,0.5042880177497864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,16384,0.6619200110435486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,5120,0.22777600586414337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,7168,0.30668801069259644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,4096,0.18889600038528442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,2048,0.10886400192975998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,2560,0.12857599556446075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,3072,0.1488640010356903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,3584,0.1690240055322647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,256,0.03791999816894531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,512,0.04835199937224388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,65536,3.9990720748901367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,1024,0.06860800087451935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,1536,0.088639996945858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,128,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,8192,0.2767679989337921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,10240,0.34060800075531006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,12288,0.4041599929332733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,16384,0.5296639800071716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,5120,0.1833599954843521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,4096,0.15142400562763214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,7168,0.24624000489711761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,2560,0.104032002389431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,2048,0.08790399879217148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,3072,0.12009599804878235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,3584,0.135903999209404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,256,0.031328000128269196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,512,0.03929600119590759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,1024,0.05648000165820122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,1536,0.07199999690055847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,128,0.02703999914228916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,65536,3.49289608001709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,8192,0.2096640020608902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,10240,0.25785601139068604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,12288,0.3049600124359131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,16384,0.39923200011253357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,4096,0.11481600254774094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,5120,0.13897599279880524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,7168,0.18704000115394592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,2048,0.06761600077152252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,2560,0.07993599772453308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,3584,0.10390400141477585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,3072,0.09177599847316742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,512,0.03126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,256,0.02521600015461445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,1024,0.04383999854326248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,1536,0.05558399856090546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,128,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,65536,2.9334399700164795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,10240,0.173567995429039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,8192,0.14192000031471252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,12288,0.20582400262355804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,16384,0.26812800765037537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,4096,0.0788159966468811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,5120,0.0947519987821579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,7168,0.12649600207805634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,3584,0.07068800181150436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,3072,0.06284800171852112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,2560,0.05503999814391136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,2048,0.047359999269247055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,512,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,1024,0.03062400035560131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,1536,0.038784001022577286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,256,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,128,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,65536,2.3748159408569336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,8192,0.0745600014925003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,10240,0.09036800265312195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,12288,0.10639999806880951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,16384,0.13753600418567657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,4096,0.04255999997258186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,5120,0.05075199902057648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,7168,0.06675200164318085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,3584,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,2560,0.032416000962257385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,3072,0.03766399994492531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,2048,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,65536,1.800096035003662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,1536,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,256,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,512,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,1024,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,128,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,8192,0.05411199852824211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,10240,0.06611199676990509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,12288,0.07699199765920639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,16384,0.10175999999046326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,5120,0.03673600032925606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,4096,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,7168,0.04825599864125252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,65536,1.1793279647827148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,2048,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,2560,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,3584,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,3072,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,256,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,1536,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,512,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,128,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,1024,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,8192,0.03580800071358681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,10240,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,12288,0.047968000173568726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,16384,0.059328000992536545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,5120,0.02470399998128414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,4096,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,7168,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,65536,0.5934399962425232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,1536,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,2048,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,2560,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,3584,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,3072,0.01759999990463257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,65536,0.4282880127429962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,65536,0.23174400627613068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,5120,2.7878079414367676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,7168,3.7830400466918945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,8192,4.275392055511475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,4096,2.2880959510803223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,10240,5.273536205291748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,3072,1.7911360263824463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,3584,2.0363519191741943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,2560,1.5415040254592896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,2048,1.2715200185775757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,128,0.30112001299858093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,256,0.38659200072288513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,512,0.5117759704589844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,1024,0.7650240063667297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,1536,1.0213439464569092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,12288,6.265696048736572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,7168,0.9653440117835999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,8192,1.0895999670028687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,10240,1.3431999683380127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,12288,1.5956480503082275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,4096,0.5849599838256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,3584,0.5230399966239929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,5120,0.712831974029541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,3072,0.45983999967575073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,16384,8.249919891357422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,16384,2.099263906478882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,1536,0.26425600051879883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,512,0.1361600011587143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,256,0.10275200009346008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,2560,0.39555200934410095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,2048,0.3296320140361786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,1024,0.1979839950799942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,128,0.08224000036716461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,7168,0.7259839773178101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,8192,0.8197119832038879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,10240,1.0092799663543701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,12288,1.198464035987854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,4096,0.44092801213264465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,5120,0.5362240076065063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,3584,0.39392000436782837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,3072,0.3463680148124695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,16384,1.576416015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,2560,0.2985599935054779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,2048,0.2502079904079437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,1536,0.2008640021085739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,256,0.07945600152015686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,512,0.10409600287675858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,1024,0.1523520052433014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,128,0.06380800157785416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,8192,0.6835839748382568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,10240,0.8422080278396606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,12288,0.9997119903564453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,16384,1.3143680095672607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,5120,0.4480319917201996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,7168,0.6056320071220398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,4096,0.3686400055885315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,2048,0.20979200303554535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,2560,0.24979199469089508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,3072,0.28998398780822754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,3584,0.3298240005970001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,512,0.08841600269079208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,256,0.0674239993095398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,1024,0.12863999605178833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,1536,0.16947199404239655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,128,0.054655998945236206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,10240,0.6750400066375732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,12288,0.8010560274124146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,16384,1.052351951599121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,7168,0.4859839975833893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,8192,0.5476800203323364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,5120,0.3592959940433502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,3584,0.2645759880542755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,4096,0.29631999135017395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,3072,0.23280000686645508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,2048,0.16886399686336517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,2560,0.2006720006465912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,1536,0.1363839954137802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,512,0.0716480016708374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,1024,0.10367999970912933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,256,0.05571199953556061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,128,0.04569600149989128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,65536,8.434432029724121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,12288,0.7024959921836853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,65536,6.726848125457764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,16384,0.922432005405426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,10240,0.592415988445282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,7168,0.4272960126399994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,8192,0.48047998547554016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,4096,0.2605760097503662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,5120,0.31539198756217957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,65536,5.140672206878662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,3072,0.2043839991092682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,2048,0.14819200336933136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,3584,0.23289600014686584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,2560,0.17696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,256,0.05011200159788132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,512,0.06355199962854385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,1536,0.1202239990234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,128,0.04092799872159958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,1024,0.09161599725484848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,8192,0.3452160060405731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,10240,0.42505601048469543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,12288,0.5036799907684326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,65536,4.117599964141846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,16384,0.6606400012969971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,7168,0.30697599053382874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,5120,0.22752000391483307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,4096,0.18777599930763245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,3584,0.16860799491405487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,1536,0.08777599781751633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,2048,0.10822399705648422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,2560,0.12787200510501862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,3072,0.1483519971370697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,128,0.03139200061559677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,512,0.047359999269247055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,256,0.03811199963092804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,1024,0.0679360032081604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,8192,0.2768639922142029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,10240,0.34092798829078674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,12288,0.4036799967288971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,16384,0.5300800204277039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,7168,0.24665600061416626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,5120,0.1828480064868927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,4096,0.15068799257278442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,2048,0.08777599781751633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,2560,0.1035199984908104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,3072,0.11952000111341476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,3584,0.13516800105571747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,256,0.031136000528931618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,512,0.03961599990725517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,1024,0.05536000058054924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,1536,0.07088000327348709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,128,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,65536,3.8920319080352783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,10240,0.3404799997806549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,12288,0.4033600091934204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,8192,0.27721598744392395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,16384,0.5301120281219482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,5120,0.18297599256038666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,4096,0.15059199929237366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,7168,0.24566400051116943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,3584,0.1361600011587143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,3072,0.11955200135707855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,2048,0.08764799684286118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,2560,0.1032319962978363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,1536,0.07142399996519089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,1024,0.055135998874902725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,512,0.038975998759269714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,256,0.028991999104619026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,128,0.02470399998128414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,65536,2.5550079345703125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,8192,0.21011200547218323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,10240,0.25814399123191833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,12288,0.3049600124359131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,16384,0.3991039991378784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,4096,0.11497599631547928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,65536,2.0452160835266113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,5120,0.13916799426078796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,7168,0.1863359957933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,3584,0.10371199995279312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,2048,0.06719999760389328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,2560,0.07958400249481201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,3072,0.0915519967675209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,1536,0.055615998804569244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,128,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,256,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,512,0.031168000772595406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,1024,0.04342399910092354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,8192,0.2093760073184967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,10240,0.25705599784851074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,12288,0.3044160008430481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,16384,0.3988800048828125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,7168,0.1863359957933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,4096,0.11481600254774094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,5120,0.13849599659442902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,65536,2.04531192779541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,1536,0.05552000179886818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,2048,0.06719999760389328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,3584,0.10300800204277039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,3072,0.09100800007581711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,2560,0.0788159966468811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,128,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,256,0.023072000592947006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,512,0.030079999938607216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,1024,0.043455999344587326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,8192,0.14192000031471252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,10240,0.17315199971199036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,12288,0.20479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,16384,0.26764801144599915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,4096,0.07833600044250488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,5120,0.09462399780750275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,7168,0.12720000743865967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,65536,1.53603196144104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,2048,0.04726399853825569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,3584,0.07072000205516815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,2560,0.05484800040721893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,3072,0.06268800050020218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,1536,0.039455998688936234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,512,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,1024,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,256,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,128,0.017791999503970146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,8192,0.1414719969034195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,10240,0.17363199591636658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,12288,0.20457600057125092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,16384,0.26767998933792114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,4096,0.07817599922418594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,5120,0.09452799707651138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,7168,0.12611199915409088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,65536,1.535423994064331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,3584,0.07088000327348709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,2048,0.04464000090956688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,2560,0.05385600030422211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,3072,0.06259199976921082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,1536,0.03721600025892258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,128,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,256,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,1024,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,512,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,8192,0.07379200309515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,10240,0.08956799656152725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,12288,0.10569600015878677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,16384,0.13689599931240082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,4096,0.04390399903059006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,5120,0.05023999884724617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,7168,0.06639999896287918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,65536,1.0257279872894287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,3584,0.03920000046491623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,1536,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,2048,0.026655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,2560,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,3072,0.03465599939227104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,128,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,512,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,256,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,1024,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,8192,0.05305600166320801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,10240,0.06483200192451477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,12288,0.07555200159549713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,16384,0.09775999933481216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,4096,0.03049599938094616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,7168,0.04771199822425842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,5120,0.036607999354600906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,65536,1.1180800199508667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,3584,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,2048,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,1536,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,2560,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,3072,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,1024,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,128,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,512,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,256,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,8192,0.03292800113558769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,10240,0.0387520007789135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,12288,0.04483199864625931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,65536,0.5303679704666138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,16384,0.05724800005555153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,4096,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,5120,0.023584000766277313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,7168,0.02940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,3584,0.01881599985063076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,3072,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,1536,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,2048,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,1024,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,8192,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,65536,0.38655999302864075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,10240,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,12288,0.0344959981739521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,16384,0.04227200150489807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,5120,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,4096,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,3584,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,7168,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,2048,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,1536,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,3072,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,1024,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,2560,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,256,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,512,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,128,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,65536,0.2033279985189438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,65536,0.13548800349235535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,5120,1.4186880588531494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,7168,1.9250240325927734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,8192,2.1751999855041504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,4096,1.1657919883728027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,10240,2.6812798976898193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,3584,1.0383039712905884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,3072,0.9055039882659912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,2560,0.7765439748764038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,2048,0.6458560228347778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,128,0.1566080003976822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,256,0.20028799772262573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,512,0.2604160010814667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,1024,0.38867199420928955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,12288,3.1858880519866943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,1536,0.5182399749755859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,7168,0.4862399995326996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,8192,0.5479999780654907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,10240,0.6750400066375732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,12288,0.8010560274124146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,16384,4.1939520835876465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,4096,0.29555198550224304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,5120,0.35916799306869507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,3584,0.26502400636672974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,16384,1.0528639554977417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,3072,0.23260800540447235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,2048,0.16908800601959229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,1024,0.10364799946546555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,1536,0.13753600418567657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,2560,0.20070399343967438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,512,0.07135999947786331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,256,0.05648000165820122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,128,0.04588799923658371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,7168,0.3668479919433594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,8192,0.41302400827407837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,10240,0.5083519816398621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,12288,0.6036480069160461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,4096,0.2231999933719635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,5120,0.2714560031890869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,3584,0.20015999674797058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,3072,0.17660799622535706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,16384,0.7910079956054688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,512,0.055743999779224396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,1536,0.10467199981212616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,2048,0.12828800082206726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,2560,0.15228800475597382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,1024,0.08086399734020233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,256,0.04492799937725067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,128,0.03680000081658363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,8192,0.34508800506591797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,10240,0.42505601048469543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,12288,0.503167986869812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,16384,0.6606079936027527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,4096,0.18758399784564972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,7168,0.30636799335479736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,5120,0.2269439995288849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,2048,0.10838399827480316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,3072,0.14841599762439728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,2560,0.1281599998474121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,3584,0.1679680049419403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,512,0.0480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,256,0.03903999924659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,1024,0.06860800087451935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,1536,0.08860799670219421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,128,0.03209599852561951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,10240,0.34089601039886475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,12288,0.4036160111427307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,16384,0.5297279953956604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,7168,0.24620799720287323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,8192,0.2770240008831024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,5120,0.18268799781799316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,4096,0.15068799257278442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,3584,0.13548800349235535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,3072,0.11948800086975098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,65536,4.081664085388184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,2048,0.08752000331878662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,2560,0.10364799946546555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,1536,0.07184000313282013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,256,0.03248000144958496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,512,0.039744000881910324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,1024,0.05580800026655197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,128,0.02755199931561947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,65536,3.0627200603485107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,8192,0.27699199318885803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,10240,0.340256005525589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,12288,0.403328001499176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,16384,0.5292479991912842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,65536,2.55350399017334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,4096,0.15078400075435638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,5120,0.18265600502490997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,7168,0.245728000998497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,3072,0.11923199892044067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,3584,0.13526399433612823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,2560,0.10291200131177902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,2048,0.08684799820184708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,1536,0.07100799679756165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,512,0.038336001336574554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,256,0.029632000252604485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,128,0.02470399998128414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,1024,0.0551999993622303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,8192,0.20934399962425232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,10240,0.25699201226234436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,12288,0.3040960133075714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,65536,2.044095993041992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,16384,0.39846399426460266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,4096,0.11462400108575821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,7168,0.1863040030002594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,5120,0.1387840062379837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,3584,0.1032319962978363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,3072,0.0907839983701706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,1536,0.05475199967622757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,2048,0.06684800237417221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,2560,0.07903999835252762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,1024,0.04323200136423111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,512,0.029952000826597214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,256,0.02348800003528595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,128,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,8192,0.14192000031471252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,10240,0.17337599396705627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,12288,0.2048960030078888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,16384,0.26787200570106506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,4096,0.07875200361013412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,5120,0.09404800087213516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,7168,0.12636800110340118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,3584,0.07062400132417679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,3072,0.06268800050020218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,2560,0.05462399870157242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,2048,0.04726399853825569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,1536,0.03948799893260002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,512,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,1024,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,256,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,128,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,65536,2.0436160564422607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,8192,0.1416960060596466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,12288,0.20454399287700653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,10240,0.17321600019931793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,16384,0.2678399980068207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,5120,0.09417600184679031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,4096,0.07811199873685837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,65536,1.5342719554901123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,7168,0.1265600025653839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,3584,0.07078400254249573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,2048,0.04713600128889084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,3072,0.06272000074386597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,1536,0.03916800022125244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,2560,0.054687999188899994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,1024,0.030719999223947525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,128,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,512,0.023072000592947006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,256,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,8192,0.1414719969034195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,65536,1.025056004524231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,10240,0.17350399494171143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,12288,0.20467199385166168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,16384,0.2678399980068207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,5120,0.09462399780750275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,3584,0.07088000327348709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,4096,0.07843200117349625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,7168,0.1266240030527115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,3072,0.062431998550891876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,2048,0.04483199864625931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,1536,0.03612799942493439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,1024,0.027744000777602196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,2560,0.05516799911856651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,512,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,256,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,128,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,8192,0.1273919939994812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,12288,0.1918720006942749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,10240,0.1648319959640503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,65536,1.0249279737472534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,16384,0.2481279969215393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,4096,0.06585600227117538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,5120,0.09100800007581711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,7168,0.11257600039243698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,3584,0.05961599946022034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,3072,0.05455999821424484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,2048,0.0379519984126091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,2560,0.04556800052523613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,1536,0.031968001276254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,1024,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,512,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,256,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,128,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,8192,0.07356800138950348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,12288,0.10521599650382996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,10240,0.0899839997291565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,65536,1.0248639583587646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,16384,0.1366720050573349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,4096,0.043136000633239746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,5120,0.05027199909090996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,7168,0.0663679987192154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,3584,0.03907199949026108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,3072,0.034912001341581345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,2560,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,1536,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,2048,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,1024,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,512,0.014303999952971935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,256,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,128,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,8192,0.07363200187683105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,12288,0.1050880029797554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,10240,0.0894400030374527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,65536,1.0178240537643433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,16384,0.13663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,4096,0.04262400045990944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,5120,0.050655998289585114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,7168,0.06598400324583054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,3584,0.038495998829603195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,3072,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,2560,0.03033600002527237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,2048,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,1536,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,1024,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,128,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,512,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,256,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,65536,0.5152959823608398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,12288,0.07609599828720093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,10240,0.0674239993095398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,8192,0.05270399898290634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,16384,0.0979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,4096,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,5120,0.03651199862360954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,7168,0.04793599992990494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,3584,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,3072,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,1024,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,512,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,1536,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,2560,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,2048,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,256,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,128,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,65536,0.5154560208320618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,12288,0.04460800066590309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,8192,0.0326399989426136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,10240,0.03872000053524971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,16384,0.05708799883723259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,7168,0.0297279991209507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,5120,0.023231999948620796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,3584,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,4096,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,3072,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,2048,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,1024,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,65536,0.3760960102081299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,8192,0.022048000246286392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,12288,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,10240,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,16384,0.036479998379945755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,7168,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,5120,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,3584,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,4096,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,2560,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,3072,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,2048,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,1536,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,1024,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,512,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,256,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,65536,0.204352006316185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,12288,0.047648001462221146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,10240,0.026464000344276428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,8192,0.032127998769283295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,16384,0.035551998764276505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,7168,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,3584,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,5120,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,4096,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,2560,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,3072,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,1024,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,2048,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,1536,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,512,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,65536,0.11849600076675415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,65536,0.11894399672746658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,5120,0.7124159932136536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,7168,0.9668160080909729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,8192,1.090432047843933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,4096,0.5873600244522095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,3584,0.5251200199127197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,10240,1.344223976135254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,3072,0.46134400367736816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,2560,0.3970879912376404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,12288,1.5962879657745361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,2048,0.33190399408340454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,1536,0.27001601457595825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,1024,0.2056639939546585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,512,0.1416960060596466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,256,0.1061440035700798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,128,0.08454400300979614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,7168,0.2459840029478073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,8192,0.27724799513816833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,10240,0.3407360017299652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,16384,2.1005120277404785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,12288,0.40384000539779663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,16384,0.5301120281219482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,5120,0.1828799992799759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,4096,0.1510079950094223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,3584,0.13619199395179749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,2560,0.10416000336408615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,3072,0.12015999853610992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,256,0.03315199911594391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,512,0.0414079986512661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,128,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,1024,0.056223999708890915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,1536,0.07199999690055847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,2048,0.08819200098514557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,7168,0.18620799481868744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,8192,0.20924800634384155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,10240,0.25724801421165466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,12288,0.30451199412345886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,4096,0.11503999680280685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,5120,0.1387840062379837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,16384,0.3989120125770569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,3072,0.09180799871683121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,3584,0.10364799946546555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,2048,0.06761600077152252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,2560,0.0796160027384758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,1024,0.04342399910092354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,512,0.03299200162291527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,1536,0.05615999922156334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,256,0.026335999369621277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,128,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,8192,0.2091519981622696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,12288,0.3044160008430481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,10240,0.2571519911289215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,16384,0.39897599816322327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,4096,0.11456000059843063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,5120,0.1383039951324463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,7168,0.18572799861431122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,3584,0.10294400155544281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,2048,0.06700800359249115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,3072,0.09136000275611877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,2560,0.0791039988398552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,512,0.03167999908328056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,256,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,1024,0.04339199885725975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,1536,0.05548800155520439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,128,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,12288,0.2051520049571991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,65536,2.0443201065063477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,10240,0.17350399494171143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,16384,0.26787200570106506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,5120,0.09424000233411789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,7168,0.12598399817943573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,8192,0.14127999544143677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,4096,0.07872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,65536,1.5349119901657104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,3584,0.07119999825954437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,2048,0.04710400104522705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,2560,0.054816000163555145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,3072,0.06323199719190598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,1536,0.03929600119590759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,1024,0.030912000685930252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,512,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,256,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,128,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,8192,0.14156800508499146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,12288,0.20508800446987152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,10240,0.1730560064315796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,65536,1.5344640016555786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,16384,0.2680000066757202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,4096,0.07833600044250488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,3584,0.0708480030298233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,5120,0.09427200257778168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,7168,0.12591999769210815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,3072,0.06297600269317627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,2048,0.04710400104522705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,1536,0.03984000161290169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,1024,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,2560,0.05459199845790863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,512,0.024447999894618988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,128,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,256,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,65536,1.025439977645874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,8192,0.12745599448680878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,12288,0.19516800343990326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,10240,0.16550399363040924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,16384,0.2553600072860718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,4096,0.07478400319814682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,5120,0.0857279971241951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,7168,0.1133119985461235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,3584,0.06124800071120262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,3072,0.05974400043487549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,2560,0.04662400111556053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,1024,0.024639999493956566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,2048,0.03782400116324425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,1536,0.03187200054526329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,256,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,128,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,512,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,8192,0.07369600236415863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,12288,0.10559999942779541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,10240,0.08963199704885483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,16384,0.13705599308013916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,65536,1.0253759622573853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,4096,0.04214400053024292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,5120,0.05023999884724617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,7168,0.06620799750089645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,3584,0.038176000118255615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,3072,0.03481600061058998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,2560,0.030848000198602676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,2048,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,1536,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,1024,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,512,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,256,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,128,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,65536,1.0248639583587646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,8192,0.07379200309515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,12288,0.1058880016207695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,10240,0.08982399851083755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,16384,0.13705599308013916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,5120,0.049984000623226166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,3584,0.03884800150990486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,4096,0.04342399910092354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,7168,0.06623999774456024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,3072,0.03446400165557861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,2560,0.03030399978160858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,2048,0.026335999369621277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,1536,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,1024,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,512,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,256,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,128,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,65536,0.5158079862594604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,12288,0.10550399869680405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,8192,0.0735040009021759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,10240,0.08959999680519104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,7168,0.0655359998345375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,16384,0.13702400028705597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,4096,0.04278400167822838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,3584,0.03855999931693077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,3072,0.034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,5120,0.051711998879909515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,2560,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,1536,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,2048,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,512,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,1024,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,256,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,128,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,65536,0.5155199766159058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,12288,0.10518400371074677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,8192,0.07248000055551529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,10240,0.08985599875450134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,16384,0.13663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,7168,0.06384000182151794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,4096,0.03961599990725517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,5120,0.047359999269247055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,3584,0.03587200120091438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,3072,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,2048,0.024607999250292778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,2560,0.028672000393271446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,1536,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,1024,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,512,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,256,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,128,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,65536,0.5156159996986389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,12288,0.07872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,8192,0.052799999713897705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,10240,0.07241600006818771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,7168,0.047968000173568726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,16384,0.09772799909114838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,3584,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,4096,0.03033600002527237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,5120,0.03657599911093712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,2560,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,3072,0.025280000641942024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,2048,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,1536,0.01651199907064438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,1024,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,512,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,256,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,128,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,65536,0.515392005443573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,12288,0.0634239986538887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,8192,0.04467200115323067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,7168,0.03936000168323517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,10240,0.05503999814391136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,16384,0.07926400005817413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,4096,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,3584,0.023840000852942467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,3072,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,5120,0.030592000111937523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,2560,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,2048,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,1536,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,1024,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,512,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,256,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,128,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,65536,0.39267200231552124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,12288,0.044895999133586884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,10240,0.03859199956059456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,7168,0.029152000322937965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,8192,0.03219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,16384,0.056703999638557434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,5120,0.023296000435948372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,3584,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,4096,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,3072,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,2560,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,1536,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,1024,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,2048,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,512,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,256,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,65536,0.3440319895744324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,12288,0.028351999819278717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,10240,0.026016000658273697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,8192,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,7168,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,16384,0.037087999284267426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,5120,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,3584,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,4096,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,3072,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,2560,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,2048,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,65536,0.20188799500465393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,1536,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,1024,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,12288,0.028192000463604927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,8192,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,10240,0.03728000074625015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,7168,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,16384,0.03481600061058998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,3584,0.013535999692976475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,3072,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,5120,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,4096,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,2560,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,1024,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,1536,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,2048,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,65536,0.11734399944543839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,256,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,12288,0.04335999861359596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,8192,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,10240,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,7168,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,16384,0.055743999779224396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,4096,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,5120,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,3584,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,3072,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,2560,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,1536,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,1024,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,2048,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,512,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,65536,0.11635199934244156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,8192,0.8196160197257996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,12288,1.1990400552749634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,65536,0.11680000275373459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,5120,0.5361599922180176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,3584,0.3948799967765808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,3072,0.34860798716545105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,4096,0.4410240054130554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,1536,0.2043839991092682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,7168,0.7265599966049194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,512,0.11276800185441971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,1024,0.1573760062456131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,256,0.08259200304746628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,2560,0.30105599761009216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,2048,0.2519359886646271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,128,0.06639999896287918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,12288,0.3044480085372925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,10240,1.0098240375518799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,8192,0.20924800634384155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,5120,0.13872000575065613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,10240,0.25702399015426636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,7168,0.18614399433135986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,2560,0.08006399869918823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,3072,0.09171199798583984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,16384,0.39926400780677795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,4096,0.1149120032787323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,3584,0.10361599922180176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,2048,0.06831999868154526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,1536,0.05539200082421303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,256,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,512,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,1024,0.04447999969124794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,128,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,16384,1.5769280195236206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,8192,0.20975999534130096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,10240,0.2568640112876892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,12288,0.3044160008430481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,7168,0.18598400056362152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,16384,0.3986240029335022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,3584,0.10300800204277039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,4096,0.11475200206041336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,5120,0.13795199990272522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,3072,0.09136000275611877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,2048,0.06707199662923813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,2560,0.07868800312280655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,1024,0.042847998440265656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,512,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,1536,0.055135998874902725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,256,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,128,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,10240,0.1732800006866455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,8192,0.1414400041103363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,12288,0.20483200252056122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,7168,0.1260479986667633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,16384,0.26812800765037537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,5120,0.0944959968328476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,4096,0.07846400141716003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,3584,0.07129599899053574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,3072,0.06329599767923355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,2560,0.05516799911856651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,2048,0.04710400104522705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,1536,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,512,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,1024,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,128,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,256,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,10240,0.17292800545692444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,12288,0.20476800203323364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,65536,1.5356160402297974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,16384,0.26796799898147583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,7168,0.12620800733566284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,8192,0.1414400041103363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,5120,0.0942080020904541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,65536,1.5343040227890015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,4096,0.07823999971151352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,3584,0.07052800059318542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,3072,0.06294400244951248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,2048,0.04652800038456917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,2560,0.05443200096487999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,1536,0.03855999931693077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,1024,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,256,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,128,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,512,0.023840000852942467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,12288,0.20502400398254395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,8192,0.14124800264835358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,10240,0.17312000691890717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,16384,0.26819199323654175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,4096,0.07798399776220322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,3584,0.07056000083684921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,5120,0.09433600306510925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,7168,0.12591999769210815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,65536,1.0259519815444946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,2560,0.05471999943256378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,1536,0.038975998759269714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,3072,0.062272001057863235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,2048,0.046592000871896744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,1024,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,512,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,256,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,128,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,12288,0.1629759967327118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,10240,0.14057600498199463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,8192,0.11830399930477142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,5120,0.07849600166082382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,16384,0.22633600234985352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,7168,0.09382399916648865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,65536,1.0252480506896973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,4096,0.06592000275850296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,3584,0.051552001386880875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,2560,0.04451199993491173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,2048,0.032416000962257385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,3072,0.05145600065588951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,1024,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,256,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,512,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,1536,0.027648000046610832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,128,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,12288,0.10505600273609161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,65536,0.5157439708709717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,8192,0.07337599992752075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,10240,0.08956799656152725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,5120,0.049695998430252075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,7168,0.06576000154018402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,16384,0.1369280070066452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,4096,0.041760001331567764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,3584,0.03811199963092804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,2560,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,2048,0.026623999699950218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,3072,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,1536,0.022304000332951546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,65536,1.0254720449447632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,1024,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,512,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,256,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,128,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,7168,0.06560000032186508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,16384,0.13699199259281158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,8192,0.07344000041484833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,12288,0.10521599650382996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,10240,0.08931200206279755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,4096,0.042527999728918076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,2560,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,5120,0.04992000013589859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,3584,0.038656000047922134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,3072,0.034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,512,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,1536,0.02223999984562397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,1024,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,256,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,2048,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,128,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,16384,0.13676799833774567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,12288,0.10518400371074677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,65536,0.515392005443573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,10240,0.08924800157546997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,8192,0.07331199944019318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,4096,0.0424639992415905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,3072,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,3584,0.03827200084924698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,5120,0.0496320016682148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,7168,0.06627199798822403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,2560,0.0306560005992651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,1536,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,2048,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,1024,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,512,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,256,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,128,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,65536,0.5152959823608398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,12288,0.10499200224876404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,10240,0.08931200206279755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,8192,0.05711999908089638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,16384,0.1367039978504181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,7168,0.05984000116586685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,4096,0.03824000060558319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,5120,0.039264000952243805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,3584,0.03488000109791756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,3072,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,2560,0.027648000046610832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,2048,0.02396799996495247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,1536,0.020735999569296837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,1024,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,512,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,256,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,128,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,65536,0.515392005443573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,8192,0.052639998495578766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,7168,0.04729599878191948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,12288,0.07532799988985062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,10240,0.0652799978852272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,16384,0.09731200337409973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,3072,0.025151999667286873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,4096,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,5120,0.03619199991226196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,3584,0.02783999964594841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,2560,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,1024,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,1536,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,2048,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,512,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,256,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,65536,0.5153599977493286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,12288,0.05936000123620033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,7168,0.03936000168323517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,10240,0.05289600044488907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,8192,0.043935999274253845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,16384,0.08256000280380249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,4096,0.026016000658273697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,3072,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,3584,0.024064000695943832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,5120,0.029920000582933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,2560,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,2048,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,512,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,1536,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,1024,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,65536,0.37459200620651245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,128,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,12288,0.04438399896025658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,10240,0.03840000182390213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,7168,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,8192,0.032127998769283295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,16384,0.05657599866390228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,3584,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,4096,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,5120,0.023264000192284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,2560,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,2048,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,1024,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,1536,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,256,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,65536,0.30988800525665283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,12288,0.028192000463604927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,8192,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,10240,0.02876799926161766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,7168,0.019872000440955162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,16384,0.03551999852061272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,4096,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,5120,0.016448000445961952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,3072,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,3584,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,2560,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,2048,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,1024,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,1536,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,65536,0.20147199928760529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,10240,0.03731200098991394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,7168,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,12288,0.04281599819660187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,8192,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,65536,0.11615999788045883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,16384,0.05459199845790863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,4096,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,5120,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,3584,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,3072,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,2560,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,2048,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,1024,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,1536,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,12288,0.0424639992415905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,7168,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,8192,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,10240,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,5120,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,16384,0.034591998904943466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,4096,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,3584,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,3072,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,2560,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,2048,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,1536,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,1024,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,512,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,65536,0.11631999909877777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,65536,0.11648000031709671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,5120,0.3601599931716919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,7168,0.4872959852218628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,8192,0.5485439896583557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,4096,0.296671986579895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,3584,0.2655999958515167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,10240,0.6760320067405701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,3072,0.2340800017118454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,2048,0.171424001455307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,12288,0.8024640083312988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,2560,0.20294399559497833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,1024,0.10902400314807892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,128,0.04835199937224388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,256,0.05942400172352791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,512,0.07814399898052216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,1536,0.1398400068283081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,16384,1.0535039901733398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,7168,0.1263040006160736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,8192,0.14176000654697418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,10240,0.1738560050725937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,12288,0.20479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,5120,0.09459199756383896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,4096,0.07846400141716003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,16384,0.2677760124206543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,3584,0.07110399752855301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,2560,0.05510399863123894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,3072,0.06310400366783142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,256,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,2048,0.047488000243902206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,512,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,1024,0.032416000962257385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,128,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,1536,0.03920000046491623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,8192,0.14124800264835358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,7168,0.12585599720478058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,12288,0.20476800203323364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,10240,0.17324799299240112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,16384,0.2677760124206543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,4096,0.07811199873685837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,5120,0.09408000111579895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,3584,0.07062400132417679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,3072,0.06268800050020218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,2560,0.0551999993622303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,1024,0.029983999207615852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,1536,0.03743999823927879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,2048,0.047200001776218414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,512,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,256,0.016767999157309532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,128,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,12288,0.20374399423599243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,8192,0.13148799538612366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,10240,0.1730239987373352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,16384,0.25753599405288696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,5120,0.09216000139713287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,4096,0.07075200229883194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,7168,0.11785600334405899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,3584,0.06435199826955795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,65536,1.0260800123214722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,3072,0.056352000683546066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,2560,0.04870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,2048,0.04342399910092354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,1536,0.036159999668598175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,256,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,512,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,1024,0.0261439997702837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,128,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,12288,0.10553599894046783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,10240,0.08982399851083755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,8192,0.07340800017118454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,65536,1.0253759622573853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,16384,0.13676799833774567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,5120,0.05004800111055374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,4096,0.04201599955558777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,3584,0.03843199834227562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,7168,0.0660799965262413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,3072,0.03452799841761589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,2560,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,2048,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,1024,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,1536,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,512,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,256,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,128,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,65536,1.0255680084228516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,12288,0.10524799674749374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,8192,0.0737600028514862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,10240,0.08979199826717377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,16384,0.1369280070066452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,7168,0.06585600227117538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,5120,0.04995200037956238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,3584,0.03855999931693077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,4096,0.042399998754262924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,3072,0.0342399999499321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,2560,0.030912000685930252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,2048,0.027168000116944313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,1536,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,1024,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,512,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,128,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,256,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,65536,0.5163840055465698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,8192,0.07046400010585785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,12288,0.10499200224876404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,10240,0.08953599631786346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,16384,0.13676799833774567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,7168,0.06304000318050385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,3584,0.03577600046992302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,5120,0.04694399982690811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,4096,0.039744000881910324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,2560,0.028575999662280083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,3072,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,2048,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,1536,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,1024,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,512,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,128,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,256,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,65536,0.5160959959030151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,8192,0.05315199866890907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,12288,0.08940800279378891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,10240,0.07171200215816498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,16384,0.09904000163078308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,7168,0.05180799961090088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,3584,0.028575999662280083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,3072,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,4096,0.031136000528931618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,5120,0.04182400181889534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,2560,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,2048,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,256,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,1536,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,512,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,1024,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,65536,0.5156159996986389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,128,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,8192,0.05257600173354149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,12288,0.0791039988398552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,7168,0.04800000041723251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,10240,0.06668800115585327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,16384,0.09708800166845322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,5120,0.03836800158023834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,3584,0.028095999732613564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,3072,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,4096,0.03200000151991844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,2560,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,65536,0.4096960127353668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,2048,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,1536,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,512,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,1024,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,256,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,128,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,12288,0.06716799736022949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,8192,0.04492799937725067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,10240,0.055424001067876816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,7168,0.04047999903559685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,16384,0.08323200047016144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,5120,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,3072,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,4096,0.0261439997702837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,3584,0.02457600086927414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,2560,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,256,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,2048,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,1536,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,1024,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,512,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,65536,0.38790398836135864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,128,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,12288,0.06425599753856659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,7168,0.03936000168323517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,10240,0.05455999821424484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,8192,0.04342399910092354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,16384,0.08246400207281113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,5120,0.030848000198602676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,4096,0.02627200074493885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,3072,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,3584,0.023711999878287315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,2560,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,65536,0.3218879997730255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,1536,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,2048,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,1024,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,512,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,256,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,128,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,12288,0.04476799815893173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,8192,0.03209599852561951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,10240,0.03843199834227562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,7168,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,16384,0.057023998349905014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,5120,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,3072,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,4096,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,3584,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,2560,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,2048,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,1024,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,512,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,65536,0.30588799715042114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,256,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,12288,0.04435199871659279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,7168,0.02921600081026554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,8192,0.031968001276254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,10240,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,16384,0.056703999638557434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,4096,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,5120,0.023104000836610794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,3584,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,3072,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,65536,0.20243200659751892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,2048,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,256,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,1536,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,1024,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,512,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,12288,0.02812799997627735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,8192,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,7168,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,10240,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,16384,0.03551999852061272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,3584,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,4096,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,5120,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,3072,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,2560,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,65536,0.2017280012369156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,2048,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,512,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,1536,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,128,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,7168,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,8192,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,12288,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,10240,0.036479998379945755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,16384,0.0350399985909462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,4096,0.014495999552309513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,3584,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,5120,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,3072,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,2560,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,65536,0.11689600348472595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,2048,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,1536,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,1024,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,12288,0.028063999488949776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,8192,0.03001599945127964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,7168,0.01942400075495243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,10240,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,16384,0.03481600061058998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,5120,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,4096,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,3584,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,3072,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,65536,0.11638399958610535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,2560,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,1536,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,2048,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,1024,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,512,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,12288,0.042399998754262924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,10240,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,8192,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,7168,0.02751999907195568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,16384,0.03471999987959862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,5120,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,4096,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,3584,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,3072,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,65536,0.11667200177907944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,2560,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,2048,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,1536,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,1024,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,65536,0.11587200313806534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,5120,0.2740800082683563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,7168,0.4830079972743988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,4096,0.22438399493694305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,8192,0.4137600064277649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,3584,0.26281601190567017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,10240,0.5110080242156982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,3072,0.2322240024805069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,2048,0.13091200590133667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,2560,0.20095999538898468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,12288,0.6061440110206604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,1536,0.137472003698349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,512,0.07651200145483017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,128,0.03903999924659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,1024,0.08470399677753448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,256,0.04681599885225296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,7168,0.12671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,16384,0.7923840284347534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,8192,0.1416960060596466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,10240,0.17417599260807037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,12288,0.20521600544452667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,16384,0.2677119970321655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,4096,0.07872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,3584,0.07078400254249573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,5120,0.09446399658918381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,3072,0.0626240000128746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,2560,0.05516799911856651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,2048,0.04681599885225296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,256,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,128,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,1536,0.03884800150990486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,1024,0.03222399950027466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,512,0.024447999894618988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,7168,0.12588800489902496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,8192,0.14105600118637085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,12288,0.20502400398254395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,10240,0.1732800006866455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,4096,0.07795199751853943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,16384,0.2674559950828552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,5120,0.09417600184679031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,3584,0.0700799971818924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,3072,0.06233600154519081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,2560,0.054496001452207565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,2048,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,1024,0.028960000723600388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,512,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,1536,0.036031998693943024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,128,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,256,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,12288,0.17420800030231476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,7168,0.10169599950313568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,10240,0.08988799899816513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,8192,0.11327999830245972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,16384,0.219200000166893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,3584,0.054655998945236206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,4096,0.060736000537872314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,5120,0.0769599974155426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,3072,0.055904000997543335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,65536,1.0255359411239624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,2560,0.043296001851558685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,2048,0.037728000432252884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,1536,0.031136000528931618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,1024,0.025599999353289604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,512,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,256,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,128,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,65536,0.5160959959030151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,8192,0.07385600358247757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,12288,0.10543999820947647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,65536,1.0251840353012085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,7168,0.06576000154018402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,10240,0.08979199826717377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,16384,0.13686400651931763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,4096,0.04227200150489807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,5120,0.050016000866889954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,3072,0.034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,2560,0.030880000442266464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,3584,0.03846399858593941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,2048,0.0272000003606081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,1024,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,1536,0.023104000836610794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,128,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,512,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,256,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,12288,0.10547199845314026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,10240,0.08947200328111649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,7168,0.06579200178384781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,8192,0.07340800017118454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,5120,0.04982399940490723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,4096,0.04214400053024292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,16384,0.13654400408267975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,3584,0.037856001406908035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,3072,0.033824000507593155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,2560,0.029920000582933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,2048,0.02691200003027916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,1536,0.02287999913096428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,256,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,512,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,1024,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,128,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,12288,0.10480000078678131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,65536,0.5157759785652161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,8192,0.05958399921655655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,10240,0.08956799656152725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,16384,0.1367039978504181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,4096,0.03824000060558319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,3584,0.03471999987959862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,7168,0.060416001826524734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,5120,0.04073600098490715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,3072,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,2560,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,1536,0.021023999899625778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,512,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,1024,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,2048,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,128,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,256,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,12288,0.07545600086450577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,65536,0.5156159996986389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,8192,0.052799999713897705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,10240,0.07148800045251846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,16384,0.09724800288677216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,5120,0.036159999668598175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,7168,0.04732799902558327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,4096,0.03062400035560131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,3584,0.02796800062060356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,3072,0.025567999109625816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,2560,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,65536,0.5157439708709717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,2048,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,1536,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,1024,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,512,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,256,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,128,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,12288,0.07548800110816956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,7168,0.04742399975657463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,8192,0.052639998495578766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,10240,0.06470400094985962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,16384,0.09737599641084671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,5120,0.03638400137424469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,4096,0.0306560005992651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,3584,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,2560,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,3072,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,65536,0.40428799390792847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,2048,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,1536,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,512,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,1024,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,256,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,128,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,12288,0.06326399743556976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,7168,0.04022400081157684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,10240,0.05318399891257286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,8192,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,16384,0.08144000172615051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,4096,0.026079999282956123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,5120,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,3584,0.024159999564290047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,3072,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,2560,0.01942400075495243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,1536,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,65536,0.3641600012779236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,2048,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,1024,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,512,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,256,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,128,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,12288,0.05241600051522255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,8192,0.03270399942994118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,7168,0.02985600009560585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,10240,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,5120,0.024320000782608986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,16384,0.057792000472545624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,4096,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,3584,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,3072,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,2048,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,2560,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,1536,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,65536,0.31407999992370605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,1024,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,256,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,512,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,128,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,12288,0.04524800181388855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,7168,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,10240,0.03827200084924698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,8192,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,16384,0.05660799890756607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,5120,0.02332800067961216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,3584,0.018688000738620758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,4096,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,2560,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,65536,0.24607999622821808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,256,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,512,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,128,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,12288,0.04464000090956688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,8192,0.03209599852561951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,10240,0.03814399987459183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,7168,0.029120000079274178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,16384,0.05648000165820122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,3584,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,5120,0.023135999217629433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,4096,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,65536,0.20214399695396423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,2048,0.01375999953597784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,256,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,512,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,12288,0.028831999748945236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,10240,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,8192,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,7168,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,16384,0.034912001341581345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,3584,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,4096,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,5120,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,3072,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,2560,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,65536,0.20108799636363983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,2048,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,1024,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,1536,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,10240,0.036448001861572266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,12288,0.02831999957561493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,8192,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,7168,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,16384,0.0350399985909462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,3584,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,4096,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,5120,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,3072,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,2560,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,65536,0.11635199934244156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,2048,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,1536,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,512,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,1024,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,12288,0.042367998510599136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,7168,0.02751999907195568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,8192,0.0297279991209507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,10240,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,16384,0.05276799947023392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,5120,0.02179200015962124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,4096,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,3584,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,65536,0.11584000289440155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,2560,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,3072,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,2048,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,1536,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,1024,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,12288,0.0424639992415905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,7168,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,8192,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,16384,0.03478400036692619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,10240,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,5120,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,3072,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,3584,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,65536,0.11635199934244156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,4096,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,2560,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,1536,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,2048,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,1024,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,65536,0.11577600240707397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,5120,0.18822400271892548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,4096,0.15347200632095337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,7168,0.25328001379966736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,8192,0.2812800109386444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,10240,0.35260799527168274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,3584,0.14287999272346497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,3072,0.12409599870443344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,2560,0.1101439967751503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,12288,0.4118080139160156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,2048,0.0923520028591156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,1536,0.07686399668455124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,512,0.044096000492572784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,128,0.02924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,1024,0.06022400036454201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,256,0.03455999866127968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,16384,0.5354560017585754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,7168,0.06896000355482101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,8192,0.07472000271081924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,12288,0.10857599973678589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,10240,0.09344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,5120,0.050912000238895416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,16384,0.14022399485111237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,4096,0.04262400045990944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,2560,0.03235200047492981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,3072,0.03593600168824196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,3584,0.041600000113248825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,2048,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,1536,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,512,0.016448000445961952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,256,0.013535999692976475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,1024,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,128,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,8192,0.07391999661922455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,7168,0.06694400310516357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,12288,0.10604800283908844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,10240,0.08988799899816513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,16384,0.13731199502944946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,3584,0.03984000161290169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,3072,0.03465599939227104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,4096,0.042399998754262924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,5120,0.050335999578237534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,2560,0.03161599859595299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,1536,0.025567999109625816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,2048,0.03017600066959858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,512,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,1024,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,256,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,128,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,12288,0.10572800040245056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,10240,0.08953599631786346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,8192,0.07171200215816498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,16384,0.1393599957227707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,65536,0.5178560018539429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,4096,0.040063999593257904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,5120,0.049056001007556915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,7168,0.06310400366783142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,3584,0.03766399994492531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,2048,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,2560,0.029632000252604485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,3072,0.03276799991726875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,1536,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,1024,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,512,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,128,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,256,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,65536,0.5171840190887451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,12288,0.09043200314044952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,8192,0.06294400244951248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,7168,0.05318399891257286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,10240,0.07804799824953079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,16384,0.10809600353240967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,5120,0.04032000154256821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,4096,0.034304000437259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,3072,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,3584,0.032607998698949814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,2560,0.027775999158620834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,2048,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,1536,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,65536,0.5163519978523254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,256,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,1024,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,512,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,128,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,12288,0.08454400300979614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,8192,0.05302400141954422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,7168,0.0525440014898777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,10240,0.08028800040483475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,16384,0.10127999633550644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,3584,0.029952000826597214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,4096,0.03049599938094616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,5120,0.04262400045990944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,3072,0.026240000501275063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,2560,0.028224000707268715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,2048,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,65536,0.43091198801994324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,256,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,512,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,1024,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,1536,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,128,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,8192,0.045152001082897186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,12288,0.06444799900054932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,10240,0.05526399984955788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,7168,0.04044799879193306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,16384,0.0846719965338707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,3072,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,4096,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,3584,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,5120,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,2560,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,2048,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,1024,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,65536,0.3975679874420166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,1536,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,512,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,128,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,8192,0.03251200169324875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,12288,0.04707200080156326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,10240,0.03855999931693077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,7168,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,16384,0.06060799956321716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,5120,0.023711999878287315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,3072,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,4096,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,3584,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,2560,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,2048,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,65536,0.3186880052089691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,1536,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,512,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,256,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,1024,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,128,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,8192,0.03219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,12288,0.045152001082897186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,7168,0.02940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,10240,0.03868800029158592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,16384,0.05673599988222122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,3072,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,3584,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,4096,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,5120,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,2560,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,65536,0.22377599775791168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,1536,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,256,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,512,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,1024,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,2048,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,128,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,8192,0.032287999987602234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,12288,0.044224001467227936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,7168,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,10240,0.038495998829603195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,16384,0.056543998420238495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,5120,0.02332800067961216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,3584,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,65536,0.20310400426387787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,4096,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,2560,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,2048,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,1536,0.01375999953597784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,512,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,128,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,256,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,1024,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,12288,0.04438399896025658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,8192,0.03219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,10240,0.03843199834227562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,7168,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,16384,0.056352000683546066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,3584,0.01945599913597107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,3072,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,4096,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,65536,0.20182399451732635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,5120,0.023903999477624893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,2560,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,2048,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,512,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,256,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,1024,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,1536,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,7168,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,12288,0.029503999277949333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,8192,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,10240,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,16384,0.03766399994492531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,65536,0.20163199305534363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,5120,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,3584,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,3072,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,4096,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,2560,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,2048,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,1536,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,1024,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,12288,0.028543999418616295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,8192,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,7168,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,10240,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,16384,0.035999998450279236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,65536,0.12028799951076508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,4096,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,3584,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,5120,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,3072,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,2560,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,2048,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,1536,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,1024,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,12288,0.04387199878692627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,7168,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,8192,0.02112000063061714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,10240,0.036768000572919846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,16384,0.05488000065088272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,65536,0.11935999989509583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,5120,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,3584,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,3072,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,4096,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,2560,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,1536,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,2048,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,12288,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,10240,0.024960000067949295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,8192,0.030368000268936157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,7168,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,16384,0.052960000932216644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,65536,0.1175680011510849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,4096,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,3584,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,5120,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,3072,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,2560,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,2048,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,1536,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,1024,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,12288,0.027807999402284622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,8192,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,7168,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,10240,0.02502400055527687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,16384,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,65536,0.11641599982976913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,5120,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,4096,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,3072,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,2560,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,3584,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,2048,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,1024,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,1536,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,128,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,12288,0.04438399896025658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,8192,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,7168,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,16384,0.05536000058054924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,10240,0.0382080003619194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,65536,0.11590400338172913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,5120,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,3584,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,4096,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,3072,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,2048,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,2560,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,1536,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,512,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,65536,0.11615999788045883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,5120,0.18595199286937714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,3584,0.13788799941539764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,7168,0.2497279942035675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,16384,0.5312640070915222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,4096,0.15225599706172943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,8192,0.2783359885215759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,3072,0.12147200107574463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,10240,0.34431999921798706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,1024,0.05920000001788139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,2048,0.09020800143480301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,2560,0.10655999928712845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,1536,0.07545600086450577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,12288,0.4076800048351288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,128,0.023135999217629433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,256,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,512,0.04342399910092354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,16384,0.13763199746608734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,12288,0.10684800148010254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,5120,0.05116799846291542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,7168,0.06710399687290192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,10240,0.0912960022687912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,8192,0.07436800003051758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,4096,0.04265600070357323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,3584,0.03872000053524971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,2048,0.028575999662280083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,3072,0.03446400165557861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,2560,0.03081599995493889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,1536,0.02595200017094612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,512,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,1024,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,256,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,128,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,12288,0.1058880016207695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,7168,0.06611199676990509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,8192,0.07344000041484833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,10240,0.0896959975361824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,16384,0.13660800457000732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,5120,0.050175998359918594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,4096,0.04211200028657913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,3584,0.038495998829603195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,3072,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,2560,0.030559999868273735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,2048,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,1024,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,1536,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,512,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,256,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,128,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,8192,0.06995200365781784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,12288,0.10550399869680405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,10240,0.08959999680519104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,16384,0.13235199451446533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,5120,0.04732799902558327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,4096,0.03903999924659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,65536,0.5187839865684509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,7168,0.06252799928188324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,2560,0.028543999418616295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,3072,0.032287999987602234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,3584,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,1536,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,2048,0.026016000658273697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,1024,0.018239999189972878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,512,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,256,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,128,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,65536,0.5174400210380554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,12288,0.08329600095748901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,8192,0.052671998739242554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,5120,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,10240,0.07088000327348709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,7168,0.05273599922657013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,16384,0.10790400207042694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,4096,0.03315199911594391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,3584,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,2048,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,3072,0.026944000273942947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,2560,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,1536,0.01775999926030636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,512,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,128,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,1024,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,256,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,65536,0.5169919729232788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,7168,0.047807998955249786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,8192,0.052799999713897705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,12288,0.07593599706888199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,10240,0.07056000083684921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,16384,0.09708800166845322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,3072,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,5120,0.03686400130391121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,3584,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,4096,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,2560,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,2048,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,65536,0.4296959936618805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,1024,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,1536,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,256,0.010751999914646149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,512,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,128,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,7168,0.03097599931061268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,12288,0.056063998490571976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,8192,0.03411199897527695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,10240,0.0488319993019104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,16384,0.06217600032687187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,5120,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,3072,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,4096,0.02287999913096428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,3584,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,2560,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,2048,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,1536,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,65536,0.4108160138130188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,1024,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,512,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,256,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,12288,0.0451200008392334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,8192,0.03251200169324875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,7168,0.02940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,10240,0.03884800150990486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,16384,0.056832000613212585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,5120,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,4096,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,3072,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,2560,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,3584,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,65536,0.25328001379966736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,1536,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,2048,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,512,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,128,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,12288,0.04473600164055824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,8192,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,10240,0.038656000047922134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,7168,0.02940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,16384,0.05648000165820122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,5120,0.023552000522613525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,65536,0.20387199521064758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,4096,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,3072,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,3584,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,2560,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,2048,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,1536,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,1024,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,512,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,12288,0.04416000097990036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,7168,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,8192,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,16384,0.05663999915122986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,10240,0.03884800150990486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,5120,0.023135999217629433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,65536,0.20508800446987152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,3072,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,2560,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,4096,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,3584,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,1536,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,2048,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,1024,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,256,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,512,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,65536,0.20160000026226044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,12288,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,7168,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,10240,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,8192,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,5120,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,16384,0.040608000010252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,4096,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,2560,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,3584,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,2048,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,1536,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,1024,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,512,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,256,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,128,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,12288,0.029503999277949333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,10240,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,8192,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,7168,0.02070399932563305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,16384,0.03673600032925606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,5120,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,4096,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,3584,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,3072,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,65536,0.13478399813175201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,2560,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,2048,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,1024,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,512,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,7168,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,12288,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,8192,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,16384,0.03705599904060364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,10240,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,65536,0.12019199877977371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,4096,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,5120,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,3072,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,3584,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,2560,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,2048,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,1024,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,1536,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,65536,0.11878400295972824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,12288,0.04464000090956688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,10240,0.03750399872660637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,7168,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,8192,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,16384,0.03494400158524513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,5120,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,4096,0.01894400082528591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,3584,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,2560,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,3072,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,2048,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,1536,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,1024,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,12288,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,7168,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,10240,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,16384,0.05385600030422211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,8192,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,5120,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,2560,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,3072,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,4096,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,3584,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,65536,0.11750400066375732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,2048,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,512,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,1536,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,12288,0.028384000062942505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,10240,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,8192,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,7168,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,16384,0.03471999987959862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,65536,0.1170559972524643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,5120,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,3584,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,3072,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,4096,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,2560,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,2048,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,1536,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,1024,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,65536,0.11689600348472595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,12288,0.04495999962091446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,16384,0.05491200089454651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,8192,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,10240,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,7168,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,5120,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,4096,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,3584,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,3072,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,2560,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,2048,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,1024,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,512,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,65536,0.11631999909877777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,7168,0.2499839961528778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,8192,0.2789759933948517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,5120,0.18505600094795227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,3072,0.12172800302505493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,4096,0.15171200037002563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,3584,0.1388159990310669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,2560,0.10540799796581268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,10240,0.3439039885997772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,2048,0.08883199840784073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,12288,0.40966400504112244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,1024,0.059039998799562454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,1536,0.07407999783754349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,256,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,128,0.022207999601960182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,512,0.04233599826693535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,8192,0.07529599964618683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,10240,0.0920960009098053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,7168,0.06815999746322632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,16384,0.5318400263786316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,12288,0.11081600189208984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,5120,0.05225599929690361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,3584,0.039744000881910324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,4096,0.04243199899792671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,3072,0.035711999982595444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,16384,0.13743999600410461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,2560,0.03171199932694435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,1024,0.022336000576615334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,256,0.013279999606311321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,2048,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,512,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,128,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,1536,0.02316799946129322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,10240,0.09097599983215332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,8192,0.07459200173616409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,7168,0.066880002617836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,12288,0.1085439994931221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,4096,0.04201599955558777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,5120,0.05084799975156784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,16384,0.13673600554466248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,3072,0.0344959981739521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,65536,0.5178239941596985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,3584,0.0382080003619194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,1024,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,2048,0.028351999819278717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,2560,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,512,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,1536,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,256,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,128,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,16384,0.14588800072669983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,8192,0.06921599805355072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,7168,0.06147199869155884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,12288,0.10684800148010254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,10240,0.09036800265312195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,3584,0.03590400144457817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,2560,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,3072,0.0315839983522892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,4096,0.04399999976158142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,5120,0.048576001077890396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,2048,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,256,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,512,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,1024,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,1536,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,8192,0.053727999329566956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,65536,0.5165119767189026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,10240,0.07452800124883652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,12288,0.08803199976682663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,16384,0.10716799646615982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,5120,0.04179200157523155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,7168,0.04896000027656555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,3584,0.02844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,4096,0.03049599938094616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,3072,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,2048,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,2560,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,1536,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,1024,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,512,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,65536,0.5160959959030151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,65536,0.4343999922275543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,12288,0.07833600044250488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,7168,0.04851200059056282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,8192,0.05379199981689453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,10240,0.0711359977722168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,5120,0.04012800008058548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,16384,0.09731200337409973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,3584,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,4096,0.03145600110292435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,2560,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,3072,0.026815999299287796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,2048,0.02191999927163124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,1536,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,1024,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,256,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,512,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,10240,0.047775998711586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,12288,0.059647999703884125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,7168,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,8192,0.0350399985909462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,16384,0.06825599819421768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,4096,0.023744000121951103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,5120,0.028543999418616295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,3584,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,3072,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,2560,0.018400000408291817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,2048,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,1536,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,1024,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,512,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,65536,0.412992000579834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,65536,0.2417600005865097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,12288,0.04678399860858917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,16384,0.05644800141453743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,8192,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,10240,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,5120,0.02316799946129322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,7168,0.03046399913728237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,4096,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,3584,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,2560,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,3072,0.01744000054895878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,1536,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,1024,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,12288,0.04531199857592583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,65536,0.20207999646663666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,10240,0.03923200070858002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,7168,0.03033600002527237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,8192,0.033376000821590424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,5120,0.023615999147295952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,4096,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,16384,0.05663999915122986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,3584,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,2560,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,3072,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,2048,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,1024,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,128,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,7168,0.029920000582933426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,12288,0.04505600035190582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,16384,0.05644800141453743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,10240,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,8192,0.03267199918627739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,3584,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,4096,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,5120,0.023264000192284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,2560,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,65536,0.21212799847126007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,1536,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,512,0.009855999611318111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,1024,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,128,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,65536,0.2006080001592636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,12288,0.036639999598264694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,10240,0.028063999488949776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,8192,0.03407999873161316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,5120,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,7168,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,16384,0.04118400067090988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,4096,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,3584,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,3072,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,2560,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,1536,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,2048,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,128,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,65536,0.1324159950017929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,12288,0.03232000023126602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,7168,0.020640000700950623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,10240,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,8192,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,16384,0.03545600175857544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,5120,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,4096,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,3584,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,3072,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,2560,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,2048,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,1536,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,1024,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,512,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,256,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,128,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,65536,0.11856000125408173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,12288,0.029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,7168,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,10240,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,8192,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,16384,0.034912001341581345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,5120,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,4096,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,3584,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,3072,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,2048,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,1536,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,2560,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,1024,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,256,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,128,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,512,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,12288,0.04601600021123886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,7168,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,8192,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,10240,0.03872000053524971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,16384,0.03465599939227104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,5120,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,3584,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,65536,0.1167680025100708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,4096,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,2560,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,3072,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,2048,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,1536,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,512,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,1024,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,256,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,128,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,65536,0.11644800007343292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,7168,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,12288,0.028896000236272812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,16384,0.05488000065088272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,5120,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,8192,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,10240,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,3072,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,2560,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,2048,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,4096,0.01942400075495243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,1536,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,3584,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,256,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,128,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,512,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,12288,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,65536,0.11564800143241882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,8192,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,7168,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,10240,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,16384,0.053888000547885895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,5120,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,4096,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,3584,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,2560,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,3072,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,1536,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,2048,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,512,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,256,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,128,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,12288,0.045184001326560974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,10240,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,7168,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,8192,0.02179200015962124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,16384,0.03433600068092346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,4096,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,65536,0.1157120019197464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,3072,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,5120,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,3584,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,2560,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,1536,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,512,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,256,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,2048,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,128,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,65536,0.11561600118875504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,5120,0.13657599687576294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,7168,0.1839040070772171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,8192,0.20579199492931366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,4096,0.11308799684047699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,10240,0.2518399953842163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,3584,0.09955199807882309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,12288,0.2982400059700012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,3072,0.09059199690818787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,2560,0.07958400249481201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,2048,0.06643199920654297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,512,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,256,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,1536,0.055135998874902725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,1024,0.04150399938225746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,128,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,16384,0.3968000113964081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,7168,0.06124800071120262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,12288,0.09324800223112106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,8192,0.06431999802589417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,10240,0.0799039974808693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,5120,0.047168001532554626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,16384,0.1189119964838028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,4096,0.03884800150990486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,2560,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,3072,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,3584,0.03388800099492073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,2048,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,1536,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,512,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,1024,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,128,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,256,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,12288,0.07369600236415863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,8192,0.05206400156021118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,7168,0.04854400083422661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,10240,0.0631679967045784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,16384,0.09334400296211243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,3584,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,5120,0.0363520011305809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,4096,0.030592000111937523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,3072,0.02579200081527233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,2560,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,512,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,1536,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,1024,0.014911999925971031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,2048,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,256,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,128,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,65536,0.4623680114746094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,12288,0.06931199878454208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,10240,0.059007998555898666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,8192,0.049984000623226166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,16384,0.08649600297212601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,7168,0.047680001705884933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,3584,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,5120,0.03657599911093712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,4096,0.030848000198602676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,3072,0.024607999250292778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,2560,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,65536,0.350847989320755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,1024,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,2048,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,1536,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,256,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,512,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,128,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,12288,0.05052800104022026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,10240,0.04451199993491173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,7168,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,8192,0.03481600061058998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,16384,0.06265600025653839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,5120,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,4096,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,3072,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,3584,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,2560,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,65536,0.32627201080322266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,1536,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,256,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,512,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,2048,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,1024,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,128,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,12288,0.048448000103235245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,8192,0.032896000891923904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,7168,0.033376000821590424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,10240,0.04044799879193306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,16384,0.05753599852323532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,3584,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,5120,0.025760000571608543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,3072,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,4096,0.022048000246286392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,2560,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,65536,0.24908800423145294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,2048,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,1536,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,512,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,256,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,1024,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,128,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,7168,0.0306560005992651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,12288,0.04495999962091446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,8192,0.032416000962257385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,10240,0.03907199949026108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,16384,0.056543998420238495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,3072,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,5120,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,3584,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,4096,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,65536,0.225055992603302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,2560,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,1536,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,2048,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,1024,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,512,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,256,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,12288,0.033376000821590424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,8192,0.024000000208616257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,7168,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,10240,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,16384,0.040672000497579575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,4096,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,65536,0.20291200280189514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,5120,0.021695999428629875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,2560,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,3584,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,3072,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,1536,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,2048,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,256,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,128,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,12288,0.03270399942994118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,7168,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,8192,0.02470399998128414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,10240,0.028575999662280083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,16384,0.039872001856565475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,5120,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,65536,0.1393280029296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,4096,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,3584,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,3072,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,2560,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,2048,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,512,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,1536,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,1024,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,12288,0.03033600002527237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,7168,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,10240,0.026559999212622643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,8192,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,16384,0.037408001720905304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,5120,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,4096,0.017791999503970146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,65536,0.1289920061826706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,3072,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,3584,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,2560,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,2048,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,1536,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,256,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,12288,0.02921600081026554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,8192,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,7168,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,10240,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,16384,0.03711999952793121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,5120,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,65536,0.12544000148773193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,4096,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,3072,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,3584,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,2560,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,2048,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,1536,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,1024,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,12288,0.028351999819278717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,7168,0.0307839997112751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,8192,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,10240,0.026016000658273697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,16384,0.0352960005402565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,5120,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,65536,0.1218239963054657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,3584,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,3072,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,4096,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,2560,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,2048,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,1536,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,512,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,1024,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,12288,0.02812799997627735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,8192,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,7168,0.02179200015962124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,10240,0.03846399858593941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,16384,0.056703999638557434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,65536,0.11929599940776825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,3584,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,4096,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,5120,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,3072,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,2560,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,2048,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,1536,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,1024,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,12288,0.02812799997627735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,7168,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,8192,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,10240,0.03807999938726425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,16384,0.03500799834728241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,65536,0.11852800101041794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,4096,0.020255999639630318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,3584,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,3072,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,5120,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,2560,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,2048,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,1536,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,1024,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,12288,0.04451199993491173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,8192,0.03190400078892708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,7168,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,10240,0.024639999493956566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,16384,0.04057599976658821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,65536,0.11744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,5120,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,4096,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,3584,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,3072,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,2560,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,1536,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,2048,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,1024,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,12288,0.044704001396894455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,7168,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,8192,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,10240,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,16384,0.03481600061058998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,65536,0.11683200299739838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,5120,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,4096,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,3584,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,3072,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,2560,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,2048,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,1536,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,1024,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,12288,0.04492799937725067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,7168,0.029503999277949333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,8192,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,10240,0.03843199834227562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,16384,0.03488000109791756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,65536,0.11635199934244156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,4096,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,3584,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,5120,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,2560,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,2048,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,1536,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,1024,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,65536,0.1167680025100708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,7168,0.1794240027666092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,5120,0.13622400164604187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,8192,0.20262399315834045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,4096,0.11151999980211258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,10240,0.25017601251602173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,3584,0.10025600343942642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,3072,0.08915200084447861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,12288,0.2943679988384247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,2048,0.06639999896287918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,2560,0.07868800312280655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,256,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,512,0.028192000463604927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,1536,0.05363199859857559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,1024,0.04064000025391579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,128,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,16384,0.3968000113964081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,7168,0.05833600088953972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,8192,0.06265600025653839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,12288,0.0904960036277771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,10240,0.07727999985218048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,5120,0.04614400118589401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,16384,0.11791999638080597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,3584,0.03564799949526787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,4096,0.037696000188589096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,2560,0.028736000880599022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,3072,0.032127998769283295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,2048,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,1536,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,512,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,256,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,128,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,1024,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,12288,0.07171200215816498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,10240,0.06191999837756157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,7168,0.04508800059556961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,8192,0.04950400069355965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,5120,0.03577600046992302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,16384,0.09379199892282486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,4096,0.029983999207615852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,65536,0.4166080057621002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,3072,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,3584,0.02739199995994568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,2560,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,2048,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,1536,0.01788800023496151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,1024,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,256,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,512,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,128,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,7168,0.042688000947237015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,8192,0.048448000103235245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,12288,0.06755200028419495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,10240,0.05795200169086456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,16384,0.08470399677753448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,3584,0.02659199945628643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,4096,0.028991999104619026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,3072,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,5120,0.03596799820661545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,2560,0.02179200015962124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,512,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,2048,0.018783999606966972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,1536,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,1024,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,256,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,65536,0.34118399024009705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,128,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,12288,0.047520000487565994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,10240,0.041728001087903976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,8192,0.035071998834609985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,7168,0.03161599859595299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,16384,0.06028800085186958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,4096,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,3072,0.018688000738620758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,3584,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,5120,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,2560,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,2048,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,1536,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,512,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,1024,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,65536,0.3133760094642639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,256,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,128,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,10240,0.04089599847793579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,12288,0.04639999940991402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,8192,0.03308799862861633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,7168,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,16384,0.05756799876689911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,5120,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,4096,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,3072,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,3584,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,2560,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,65536,0.23606400191783905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,2048,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,1536,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,1024,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,512,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,256,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,128,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,12288,0.04473600164055824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,8192,0.03267199918627739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,7168,0.02921600081026554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,10240,0.03830400109291077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,16384,0.05721599981188774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,4096,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,5120,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,3584,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,3072,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,65536,0.20528000593185425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,2048,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,2560,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,1536,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,1024,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,512,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,256,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,12288,0.032575998455286026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,16384,0.04064000025391579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,8192,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,7168,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,10240,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,5120,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,4096,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,3584,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,3072,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,65536,0.2016959935426712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,2560,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,2048,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,1024,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,512,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,65536,0.13436800241470337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,12288,0.031168000772595406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,8192,0.02287999913096428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,10240,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,16384,0.03936000168323517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,7168,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,4096,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,2560,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,3584,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,3072,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,5120,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,2048,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,512,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,12288,0.03046399913728237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,8192,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,10240,0.02598400041460991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,7168,0.021088000386953354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,16384,0.03788800165057182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,5120,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,4096,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,3072,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,3584,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,65536,0.12905600666999817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,2560,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,2048,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,1536,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,1024,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,12288,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,8192,0.03276799991726875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,7168,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,10240,0.02470399998128414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,16384,0.036479998379945755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,65536,0.1225920021533966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,5120,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,4096,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,3072,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,3584,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,2048,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,2560,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,1536,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,1024,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,12288,0.02816000021994114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,8192,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,16384,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,10240,0.03843199834227562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,7168,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,5120,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,65536,0.1188800036907196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,3584,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,4096,0.015552000142633915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,3072,0.017503999173641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,2560,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,1536,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,256,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,512,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,1024,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,2048,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,12288,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,8192,0.021536000072956085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,10240,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,7168,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,16384,0.036768000572919846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,65536,0.11856000125408173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,3584,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,4096,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,5120,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,3072,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,2560,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,2048,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,1536,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,1024,0.010688000358641148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,65536,0.1167680025100708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,12288,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,10240,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,7168,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,8192,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,16384,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,5120,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,4096,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,3584,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,2560,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,2048,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,3072,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,1536,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,1024,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,12288,0.044096000492572784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,16384,0.05427199974656105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,8192,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,7168,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,10240,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,5120,0.01692800037562847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,65536,0.11763200163841248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,4096,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,3072,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,3584,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,2560,0.01532800029963255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,2048,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,1536,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,1024,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,12288,0.042847998440265656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,8192,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,7168,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,10240,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,16384,0.034752000123262405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,65536,0.11695999652147293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,4096,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,5120,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,3584,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,2560,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,2048,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,1536,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,1024,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,128,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,12288,0.044256001710891724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,8192,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,7168,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,10240,0.03840000182390213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,16384,0.05539200082421303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,65536,0.11638399958610535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,4096,0.01539199985563755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,5120,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,3072,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,3584,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,2048,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,1024,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,1536,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,65536,0.20211200416088104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,7168,0.1786240041255951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,5120,0.1353279948234558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,3584,0.10070399940013885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,8192,0.20239999890327454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,4096,0.11001600325107574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,10240,0.247871994972229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,3072,0.08793599903583527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,12288,0.2947840094566345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,2560,0.07692799717187881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,2048,0.06569600105285645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,1536,0.05331199988722801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,512,0.02876799926161766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,1024,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,256,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,128,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,16384,0.38995200395584106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,12288,0.0907839983701706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,8192,0.06604799628257751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,16384,0.11219199746847153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,7168,0.057920001447200775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,10240,0.07811199873685837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,5120,0.045471999794244766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,4096,0.03721600025892258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,2048,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,3584,0.03356799855828285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,3072,0.030751999467611313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,2560,0.02739199995994568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,1536,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,1024,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,512,0.013279999606311321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,256,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,128,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,12288,0.07199999690055847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,8192,0.04873599857091904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,7168,0.044544000178575516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,10240,0.06140799820423126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,16384,0.0899519994854927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,5120,0.035711999982595444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,4096,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,3072,0.024447999894618988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,2560,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,3584,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,2048,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,1536,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,1024,0.014560000039637089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,512,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,128,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,256,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,12288,0.06735999882221222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,10240,0.05660799890756607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,65536,0.3895680010318756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,8192,0.046751998364925385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,16384,0.08537600189447403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,5120,0.035840000957250595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,4096,0.030112000182271004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,3584,0.026176000013947487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,7168,0.04249599948525429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,3072,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,2560,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,65536,0.33875200152397156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,2048,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,1024,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,1536,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,512,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,256,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,128,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,16384,0.059808000922203064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,12288,0.049984000623226166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,8192,0.03440000116825104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,10240,0.042047999799251556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,7168,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,3584,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,5120,0.02707199938595295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,4096,0.022495999932289124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,3072,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,2560,0.01744000054895878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,2048,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,1536,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,65536,0.31407999992370605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,1024,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,512,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,128,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,12288,0.04521600157022476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,8192,0.03296000137925148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,7168,0.029600000008940697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,10240,0.03888000175356865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,16384,0.057760000228881836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,3584,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,4096,0.021983999758958817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,5120,0.025631999596953392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,3072,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,2560,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,65536,0.23369599878787994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,2048,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,1536,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,1024,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,512,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,256,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,12288,0.04416000097990036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,10240,0.03824000060558319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,16384,0.05692800134420395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,7168,0.02940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,8192,0.0326399989426136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,4096,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,5120,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,3584,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,3072,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,2560,0.01651199907064438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,65536,0.20579199492931366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,2048,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,1024,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,1536,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,256,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,512,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,65536,0.2030079960823059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,8192,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,12288,0.03283200040459633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,7168,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,10240,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,5120,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,16384,0.04137599840760231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,4096,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,3584,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,3072,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,2560,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,2048,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,1024,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,12288,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,10240,0.027008000761270523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,7168,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,8192,0.022655999287962914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,16384,0.039744000881910324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,4096,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,5120,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,3072,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,3584,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,65536,0.13497599959373474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,2560,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,2048,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,1536,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,1024,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,128,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,10240,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,12288,0.029888000339269638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,7168,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,8192,0.023679999634623528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,16384,0.03830400109291077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,65536,0.12876799702644348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,5120,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,4096,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,3072,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,3584,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,2560,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,2048,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,1536,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,128,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,12288,0.030880000442266464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,8192,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,16384,0.03686400130391121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,10240,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,7168,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,4096,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,5120,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,3584,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,65536,0.1226240023970604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,3072,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,2560,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,2048,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,256,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,1024,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,1536,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,65536,0.12220799922943115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,10240,0.038336001336574554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,8192,0.033215999603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,7168,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,12288,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,5120,0.01744000054895878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,16384,0.036448001861572266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,4096,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,2560,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,3584,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,3072,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,2048,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,1536,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,1024,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,8192,0.02175999991595745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,10240,0.038176000118255615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,7168,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,12288,0.04451199993491173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,16384,0.05619199946522713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,5120,0.017791999503970146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,3072,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,4096,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,3584,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,2560,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,65536,0.11961600184440613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,2048,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,1536,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,1024,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,65536,0.11791999638080597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,5120,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,12288,0.027904000133275986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,7168,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,8192,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,10240,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,16384,0.03558399900794029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,4096,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,3072,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,2560,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,3584,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,2048,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,1536,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,1024,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,12288,0.04368000105023384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,7168,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,8192,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,10240,0.038176000118255615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,16384,0.03548799827694893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,4096,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,5120,0.01692800037562847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,3584,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,65536,0.11724799871444702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,3072,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,2560,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,2048,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,1536,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,512,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,1024,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,12288,0.04451199993491173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,8192,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,7168,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,10240,0.025087999179959297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,16384,0.0551999993622303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,65536,0.1167680025100708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,4096,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,3584,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,3072,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,5120,0.023231999948620796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,2560,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,2048,0.012415999546647072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,1536,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,512,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,1024,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,65536,0.11631999909877777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,12288,0.04399999976158142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,10240,0.0379519984126091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,8192,0.020896000787615776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,7168,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,5120,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,16384,0.03558399900794029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,4096,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,3584,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,2560,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,3072,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,2048,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,1536,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,1024,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,512,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,65536,0.11644800007343292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,5120,0.12636800110340118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,7168,0.16697600483894348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,8192,0.18758399784564972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,10240,0.23155200481414795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,4096,0.10239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,12288,0.2740800082683563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,3584,0.09305600076913834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,3072,0.08220800012350082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,2560,0.07068800181150436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,512,0.024032000452280045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,1536,0.047648001462221146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,2048,0.059487998485565186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,1024,0.036607999354600906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,128,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,256,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,16384,0.36028799414634705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,5120,0.037696000188589096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,12288,0.07513599842786789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,8192,0.05503999814391136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,10240,0.06566400080919266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,7168,0.047488000243902206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,16384,0.09881599992513657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,2560,0.024671999737620354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,2048,0.020864000543951988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,4096,0.03232000023126602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,3072,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,3584,0.0297279991209507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,1536,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,128,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,256,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,1024,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,512,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,8192,0.04416000097990036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,7168,0.039583999663591385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,12288,0.060447998344898224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,10240,0.05177599936723709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,16384,0.07715199887752533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,3584,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,3072,0.022816000506281853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,4096,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,5120,0.032416000962257385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,2560,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,2048,0.01974399946630001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,512,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,1536,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,1024,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,256,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,128,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,65536,0.36025598645210266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,12288,0.05305600166320801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,8192,0.039712000638246536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,10240,0.045504000037908554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,16384,0.06643199920654297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,7168,0.03484800085425377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,5120,0.028384000062942505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,4096,0.024992000311613083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,3072,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,3584,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,65536,0.27561599016189575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,1536,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,2048,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,2560,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,512,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,1024,0.012864000163972378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,256,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,128,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,12288,0.04371200129389763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,10240,0.03750399872660637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,8192,0.033535998314619064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,7168,0.028736000880599022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,16384,0.055296000093221664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,5120,0.024191999807953835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,4096,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,3584,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,3072,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,2560,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,65536,0.23907199501991272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,1536,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,2048,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,512,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,1024,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,12288,0.04076800122857094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,7168,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,8192,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,10240,0.03494400158524513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,16384,0.05020799860358238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,5120,0.023296000435948372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,4096,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,3584,0.01945599913597107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,3072,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,2560,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,65536,0.19308799505233765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,1536,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,1024,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,512,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,256,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,12288,0.034623999148607254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,10240,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,8192,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,7168,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,16384,0.043455999344587326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,5120,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,4096,0.018239999189972878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,3584,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,3072,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,2560,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,65536,0.17180800437927246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,2048,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,1536,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,1024,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,256,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,512,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,12288,0.031936001032590866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,7168,0.022431999444961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,8192,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,10240,0.027615999802947044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,16384,0.04057599976658821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,4096,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,5120,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,3072,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,3584,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,65536,0.14524799585342407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,2560,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,2048,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,512,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,128,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,12288,0.030880000442266464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,8192,0.023679999634623528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,10240,0.02800000086426735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,7168,0.021856000646948814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,16384,0.038336001336574554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,5120,0.01897599920630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,4096,0.021407999098300934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,3584,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,65536,0.1348479986190796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,3072,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,2560,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,2048,0.014944000169634819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,1536,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,1024,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,512,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,7168,0.022207999601960182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,8192,0.023072000592947006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,12288,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,10240,0.027264000847935677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,16384,0.03705599904060364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,5120,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,4096,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,65536,0.12758399546146393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,3584,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,3072,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,2560,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,1024,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,2048,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,512,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,12288,0.04623999819159508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,8192,0.022655999287962914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,10240,0.040063999593257904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,7168,0.03017600066959858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,16384,0.036320000886917114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,5120,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,65536,0.1250240057706833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,3584,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,4096,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,3072,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,2560,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,1536,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,2048,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,12288,0.028095999732613564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,8192,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,10240,0.03939199820160866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,7168,0.02953599952161312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,16384,0.036959998309612274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,5120,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,4096,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,65536,0.12144000083208084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,3584,0.019328000023961067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,3072,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,2560,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,2048,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,1024,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,1536,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,12288,0.028672000393271446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,7168,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,8192,0.02179200015962124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,10240,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,16384,0.05644800141453743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,4096,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,65536,0.1194240003824234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,5120,0.01744000054895878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,3584,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,3072,0.01375999953597784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,2560,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,2048,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,1024,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,1536,0.012992000207304955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,12288,0.0451200008392334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,7168,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,10240,0.025280000641942024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,8192,0.022143999114632607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,16384,0.035071998834609985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,65536,0.1186240017414093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,5120,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,4096,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,3584,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,3072,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,2560,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,2048,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,1536,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,1024,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,12288,0.04464000090956688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,10240,0.0387520007789135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,8192,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,7168,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,16384,0.03494400158524513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,65536,0.11772800236940384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,5120,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,4096,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,3584,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,3072,0.013504000380635262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,2560,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,2048,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,1536,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,1024,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,128,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,12288,0.02860799990594387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,10240,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,8192,0.03251200169324875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,7168,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,16384,0.055743999779224396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,65536,0.11654400080442429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,5120,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,4096,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,3584,0.014495999552309513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,2560,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,2048,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,3072,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,1536,0.012575999833643436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,1024,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,512,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,12288,0.044863998889923096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,7168,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,16384,0.05651199817657471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,8192,0.03190400078892708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,65536,0.11635199934244156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,10240,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,5120,0.016831999644637108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,3584,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,4096,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,3072,0.01375999953597784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,2048,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,2560,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,1536,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,128,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,1024,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,65536,0.1165120005607605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,5120,0.12364800274372101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,7168,0.16512000560760498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,10240,0.22921599447727203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,3584,0.09276799857616425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,8192,0.1855359971523285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,3072,0.08182399719953537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,4096,0.10307200253009796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,2560,0.06918399780988693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,12288,0.2725119888782501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,1536,0.04787199944257736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,2048,0.058240000158548355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,1024,0.03500799834728241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,512,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,256,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,128,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,16384,0.3571839928627014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,10240,0.06310400366783142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,12288,0.07417599856853485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,7168,0.047839999198913574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,5120,0.03564799949526787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,8192,0.053247999399900436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,16384,0.09692800045013428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,4096,0.03219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,2048,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,2560,0.02412799932062626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,3072,0.02735999971628189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,3584,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,1536,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,256,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,128,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,512,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,1024,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,12288,0.05939200147986412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,8192,0.043327998369932175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,7168,0.03884800150990486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,10240,0.050303999334573746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,16384,0.07753600180149078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,5120,0.0297279991209507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,4096,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,3072,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,3584,0.024927999824285507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,2560,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,2048,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,1536,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,1024,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,128,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,512,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,256,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,65536,0.3540480136871338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,8192,0.04016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,12288,0.05398400127887726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,10240,0.046112000942230225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,16384,0.06627199798822403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,7168,0.034591998904943466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,4096,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,5120,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,3584,0.023231999948620796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,3072,0.023391999304294586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,2560,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,2048,0.018239999189972878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,65536,0.27353599667549133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,1536,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,512,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,256,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,1024,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,12288,0.04416000097990036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,10240,0.038495998829603195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,8192,0.03171199932694435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,16384,0.054655998945236206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,7168,0.0289280004799366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,5120,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,4096,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,3584,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,3072,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,2560,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,2048,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,1536,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,1024,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,65536,0.23587200045585632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,512,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,256,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,128,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,65536,0.19247999787330627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,12288,0.040672000497579575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,7168,0.026688000187277794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,10240,0.03433600068092346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,8192,0.029152000322937965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,16384,0.05040000006556511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,4096,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,5120,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,3584,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,3072,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,2048,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,2560,0.015584000386297703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,1536,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,1024,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,256,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,512,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,128,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,65536,0.1695999950170517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,8192,0.024960000067949295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,7168,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,12288,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,10240,0.030559999868273735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,16384,0.04243199899792671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,5120,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,4096,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,3584,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,3072,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,2560,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,1024,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,1536,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,2048,0.013535999692976475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,512,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,8192,0.02489599958062172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,12288,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,7168,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,10240,0.027615999802947044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,16384,0.0387520007789135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,5120,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,4096,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,3584,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,3072,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,2560,0.013856000266969204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,65536,0.142752006649971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,2048,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,1536,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,12288,0.03110400028526783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,10240,0.02691200003027916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,8192,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,7168,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,16384,0.037248000502586365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,5120,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,4096,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,3584,0.015904000028967857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,65536,0.1324159950017929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,3072,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,2048,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,2560,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,1536,0.01196799986064434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,12288,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,10240,0.0272000003606081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,16384,0.036159999668598175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,8192,0.023231999948620796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,7168,0.024351999163627625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,5120,0.023423999547958374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,65536,0.12598399817943573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,4096,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,3584,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,2560,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,3072,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,1536,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,2048,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,1024,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,8192,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,12288,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,10240,0.0395519994199276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,16384,0.035071998834609985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,7168,0.02163200080394745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,5120,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,4096,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,65536,0.12191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,3584,0.015072000212967396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,3072,0.014271999709308147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,2560,0.013376000337302685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,2048,0.013279999606311321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,1536,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,12288,0.04492799937725067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,10240,0.03907199949026108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,8192,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,16384,0.03542400151491165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,7168,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,65536,0.1191679984331131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,5120,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,4096,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,3584,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,3072,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,2048,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,2560,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,1536,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,1024,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,12288,0.028511999174952507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,7168,0.019711999222636223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,8192,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,10240,0.03846399858593941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,16384,0.055615998804569244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,5120,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,65536,0.11737599968910217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,4096,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,3584,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,3072,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,2048,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,2560,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,1024,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,1536,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,512,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,12288,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,8192,0.021183999255299568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,10240,0.03862399980425835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,7168,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,16384,0.03488000109791756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,4096,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,65536,0.11593600362539291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,5120,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,3584,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,3072,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,2560,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,2048,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,512,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,1024,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,256,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,1536,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,12288,0.044256001710891724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,8192,0.021663999184966087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,7168,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,10240,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,16384,0.040031999349594116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,5120,0.01600000075995922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,65536,0.11526399850845337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,4096,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,3072,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,3584,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,2048,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,2560,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,1536,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,512,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,1024,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,12288,0.027744000777602196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,10240,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,7168,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,16384,0.05555199831724167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,8192,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,65536,0.1143679991364479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,4096,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,5120,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,3584,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,3072,0.01408000010997057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,2048,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,2560,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,1536,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,512,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,12288,0.04419200122356415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,10240,0.025119999423623085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,8192,0.031488001346588135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,7168,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,16384,0.05536000058054924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,5120,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,65536,0.11427199840545654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,3584,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,3072,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,4096,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,2560,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,2048,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,256,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,512,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,65536,0.1143679991364479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,5120,0.12211199849843979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,4096,0.09939199686050415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,7168,0.16409599781036377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,8192,0.18617600202560425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,12288,0.27030399441719055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,10240,0.22592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,3072,0.07779199630022049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,3584,0.08928000181913376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,2560,0.06681600213050842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,2048,0.05452800169587135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,512,0.020864000543951988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,1024,0.032607998698949814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,128,0.012095999903976917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,1536,0.04390399903059006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,256,0.015039999969303608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,16384,0.357695996761322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,7168,0.04739199951291084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,12288,0.07552000135183334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,8192,0.053727999329566956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,10240,0.062272001057863235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,5120,0.036159999668598175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,16384,0.09532800316810608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,3072,0.026079999282956123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,2560,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,3584,0.029343999922275543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,4096,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,2048,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,1536,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,1024,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,256,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,128,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,512,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,7168,0.03916800022125244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,12288,0.058720000088214874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,8192,0.04294399917125702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,10240,0.051231998950242996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,16384,0.07625599950551987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,3584,0.02473600022494793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,5120,0.03014400042593479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,3072,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,4096,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,2560,0.021023999899625778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,512,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,1536,0.016256000846624374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,1024,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,2048,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,256,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,65536,0.3522239923477173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,12288,0.05260799825191498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,10240,0.0451200008392334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,8192,0.03814399987459183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,7168,0.035232000052928925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,16384,0.06672000139951706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,5120,0.026784000918269157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,4096,0.02518399991095066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,3584,0.02348800003528595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,3072,0.022016000002622604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,2560,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,65536,0.2717440128326416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,2048,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,1536,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,1024,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,512,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,256,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,128,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,12288,0.04303999990224838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,8192,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,10240,0.03711999952793121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,7168,0.028672000393271446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,16384,0.055743999779224396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,5120,0.02300800010561943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,3584,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,4096,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,3072,0.018848000094294548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,2560,0.016767999157309532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,65536,0.2319359928369522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,2048,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,1536,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,1024,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,512,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,256,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,12288,0.040863998234272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,8192,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,10240,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,7168,0.0261439997702837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,16384,0.050464000552892685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,5120,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,4096,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,3584,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,3072,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,2560,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,65536,0.18857599794864655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,2048,0.014592000283300877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,1536,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,1024,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,512,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,128,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,12288,0.03404799848794937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,10240,0.02956799976527691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,8192,0.02521600015461445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,7168,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,16384,0.04188799858093262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,3584,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,4096,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,5120,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,3072,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,2560,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,65536,0.16921600699424744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,1536,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,2048,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,1024,0.012128000147640705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,256,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,12288,0.031168000772595406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,7168,0.03049599938094616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,8192,0.024191999807953835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,10240,0.027135999873280525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,16384,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,5120,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,3584,0.019680000841617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,4096,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,3072,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,65536,0.14179199934005737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,2560,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,2048,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,1536,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,1024,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,12288,0.031072000041604042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,8192,0.03315199911594391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,7168,0.03033600002527237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,10240,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,16384,0.038015998899936676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,5120,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,4096,0.02179200015962124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,3584,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,3072,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,65536,0.131071999669075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,2560,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,1024,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,512,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,1536,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,2048,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,12288,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,10240,0.03932800143957138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,8192,0.032896000891923904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,7168,0.029888000339269638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,16384,0.03651199862360954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,5120,0.023231999948620796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,4096,0.020927999168634415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,3584,0.01926399953663349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,3072,0.018015999346971512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,65536,0.12415999919176102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,2560,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,2048,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,1536,0.013344000093638897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,1024,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,12288,0.03465599939227104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,8192,0.032896000891923904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,7168,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,10240,0.0395519994199276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,16384,0.03641600161790848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,5120,0.02316799946129322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,3584,0.016448000445961952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,65536,0.11935999989509583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,4096,0.020896000787615776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,3072,0.01788800023496151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,2560,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,1024,0.010912000201642513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,1536,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,12288,0.044863998889923096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,7168,0.03001599945127964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,8192,0.032255999743938446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,10240,0.03846399858593941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,16384,0.036607999354600906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,5120,0.023391999304294586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,3584,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,3072,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,4096,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,65536,0.11667200177907944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,2560,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,2048,0.014336000196635723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,512,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,256,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,1536,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,1024,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,128,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,12288,0.044704001396894455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,8192,0.03187200054526329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,10240,0.03852799907326698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,7168,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,16384,0.055743999779224396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,65536,0.11615999788045883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,5120,0.022784000262618065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,3584,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,4096,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,3072,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,2048,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,1536,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,1024,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,512,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,128,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,12288,0.04447999969124794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,7168,0.02921600081026554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,8192,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,10240,0.03843199834227562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,16384,0.05507199838757515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,4096,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,5120,0.022431999444961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,65536,0.11443199962377548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,3584,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,2560,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,1536,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,2048,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,1024,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,256,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,128,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,12288,0.044415999203920364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,8192,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,10240,0.03776000067591667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,7168,0.028896000236272812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,16384,0.05516799911856651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,65536,0.11395200341939926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,4096,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,3584,0.018432000651955605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,3072,0.016896000131964684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,5120,0.022048000246286392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,2560,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,2048,0.014240000396966934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,1536,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,256,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,128,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,12288,0.04467200115323067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,8192,0.03203200176358223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,7168,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,10240,0.03846399858593941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,16384,0.05686400085687637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,65536,0.1133119985461235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,5120,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,3072,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,3584,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,4096,0.02035200037062168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,2560,0.01548799965530634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,2048,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,1536,0.012640000320971012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,12288,0.04435199871659279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,8192,0.031808000057935715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,10240,0.03791999816894531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,7168,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,16384,0.055456001311540604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,65536,0.2035840004682541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,4096,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,5120,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,3584,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,3072,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,2560,0.01568000018596649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,512,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,1536,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,1024,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,256,0.008608000352978706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,128,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,65536,0.11311999708414078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,5120,0.12038400024175644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,7168,0.1637440025806427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,8192,0.1820800006389618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,4096,0.09884800016880035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,10240,0.2248000055551529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,12288,0.26979199051856995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,3584,0.08819200098514557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,3072,0.07648000121116638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,2048,0.05302400141954422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,2560,0.06620799750089645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,1536,0.04339199885725975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,512,0.019967999309301376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,1024,0.031199999153614044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,256,0.013887999579310417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,128,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,16384,0.3523840010166168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,8192,0.05353600159287453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,5120,0.036479998379945755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,7168,0.0488319993019104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,12288,0.07395199686288834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,10240,0.06335999816656113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,16384,0.09587199985980988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,2560,0.023871999233961105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,3072,0.026176000013947487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,2048,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,3584,0.02940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,4096,0.031968001276254654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,1536,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,128,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,256,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,1024,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,512,0.011615999974310398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,7168,0.039872001856565475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,8192,0.04214400053024292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,12288,0.059328000992536545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,10240,0.04992000013589859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,16384,0.07532799988985062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,4096,0.026655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,5120,0.029184000566601753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,3584,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,3072,0.022495999932289124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,2560,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,1024,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,2048,0.018239999189972878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,1536,0.015807999297976494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,128,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,65536,0.3498240113258362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,12288,0.05270399898290634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,8192,0.038975998759269714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,16384,0.06511999666690826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,10240,0.04473600164055824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,5120,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,7168,0.03471999987959862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,4096,0.02396799996495247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,3072,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,3584,0.02287999913096428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,2560,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,65536,0.2685759961605072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,2048,0.017343999817967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,1536,0.014495999552309513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,512,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,1024,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,256,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,12288,0.04278400167822838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,8192,0.031647998839616776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,10240,0.03654399886727333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,7168,0.029600000008940697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,16384,0.05452800169587135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,4096,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,5120,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,3584,0.019840000197291374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,3072,0.018079999834299088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,2560,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,65536,0.23311999440193176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,2048,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,1536,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,1024,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,512,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,256,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,128,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,12288,0.03903999924659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,10240,0.033535998314619064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,7168,0.027295999228954315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,8192,0.028960000723600388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,16384,0.050303999334573746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,5120,0.021056000143289566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,4096,0.018912000581622124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,3072,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,3584,0.01929599978029728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,2560,0.017823999747633934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,65536,0.18940800428390503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,512,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,2048,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,1024,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,1536,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,256,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,128,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,12288,0.033215999603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,10240,0.028863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,8192,0.025599999353289604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,7168,0.02816000021994114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,16384,0.04137599840760231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,5120,0.024831999093294144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,3584,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,4096,0.018719999119639397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,3072,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,65536,0.16739200055599213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,2560,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,2048,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,1536,0.012799999676644802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,1024,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,512,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,128,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,8192,0.033663999289274216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,12288,0.0306560005992651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,7168,0.025567999109625816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,10240,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,16384,0.03903999924659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,5120,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,4096,0.021215999498963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,3584,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,3072,0.01833599992096424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,65536,0.13916799426078796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,2560,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,2048,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,1024,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,1536,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,512,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,12288,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,7168,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,8192,0.02659199945628643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,10240,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,16384,0.03753599897027016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,4096,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,5120,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,3584,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,3072,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,65536,0.12617599964141846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,2560,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,2048,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,1536,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,512,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,256,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,1024,0.011680000461637974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,12288,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,8192,0.03280000016093254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,10240,0.039103999733924866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,7168,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,16384,0.03619199991226196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,5120,0.023104000836610794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,3584,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,3072,0.01759999990463257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,4096,0.017311999574303627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,65536,0.12336000055074692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,2560,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,1536,0.012927999719977379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,2048,0.013535999692976475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,1024,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,256,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,512,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,128,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,12288,0.04473600164055824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,8192,0.025728000327944756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,10240,0.029791999608278275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,7168,0.030400000512599945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,16384,0.03542400151491165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,4096,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,65536,0.11823999881744385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,3584,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,5120,0.023104000836610794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,3072,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,2560,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,1536,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,256,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,1024,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,2048,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,12288,0.045024000108242035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,8192,0.024800000712275505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,7168,0.03049599938094616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,10240,0.026016000658273697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,16384,0.04390399903059006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,5120,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,3072,0.017472000792622566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,4096,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,3584,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,65536,0.11532799899578094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,2560,0.015968000516295433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,1536,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,2048,0.012896000407636166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,1024,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,512,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,12288,0.044704001396894455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,8192,0.031936001032590866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,10240,0.03827200084924698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,16384,0.042080000042915344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,7168,0.02924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,4096,0.020896000787615776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,5120,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,3584,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,65536,0.11379200220108032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,2560,0.015776000916957855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,3072,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,2048,0.013824000023305416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,1024,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,1536,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,256,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,128,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,12288,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,10240,0.037728000432252884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,7168,0.022975999861955643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,8192,0.031808000057935715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,16384,0.05555199831724167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,5120,0.02223999984562397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,65536,0.11283200234174728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,3072,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,3584,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,4096,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,2560,0.015424000099301338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,2048,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,1536,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,256,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,1024,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,12288,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,8192,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,7168,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,10240,0.038015998899936676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,16384,0.04047999903559685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,65536,0.1122559979557991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,5120,0.0226879995316267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,4096,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,3072,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,3584,0.018751999363303185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,2560,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,2048,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,1024,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,1536,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,512,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,128,0.007712000049650669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,12288,0.043616000562906265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,8192,0.031168000772595406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,7168,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,10240,0.03836800158023834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,16384,0.041728001087903976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,65536,0.11222399771213531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,5120,0.02208000048995018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,4096,0.020320000126957893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,3584,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,3072,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,2560,0.015519999898970127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,2048,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,512,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,1024,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,1536,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,256,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,128,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,12288,0.04403200000524521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,8192,0.0318400003015995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,7168,0.02969600073993206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,10240,0.03791999816894531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,16384,0.04124800115823746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,65536,0.13948799669742584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,5120,0.02236800082027912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,4096,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,3072,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,3584,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,2560,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,2048,0.013632000423967838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,1536,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,512,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,1024,0.011071999557316303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,256,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,65536,0.11107199639081955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,5120,0.12015999853610992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,7168,0.16371199488639832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,8192,0.18166400492191315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,4096,0.09839999675750732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,10240,0.22361600399017334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,3584,0.08726400136947632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,12288,0.26678401231765747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,3072,0.07878399640321732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,2560,0.06524799764156342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,512,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,1536,0.0427200011909008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,2048,0.053568001836538315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,1024,0.03167999908328056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,256,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,128,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,16384,0.3487679958343506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,8192,0.05478399991989136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,7168,0.04710400104522705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,12288,0.07283200323581696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,5120,0.03590400144457817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,10240,0.06268800050020218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,16384,0.09545599669218063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,2048,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,2560,0.023104000836610794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,4096,0.03142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,3072,0.024768000468611717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,3584,0.029055999591946602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,1024,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,1536,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,256,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,512,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,128,0.009216000325977802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,7168,0.038816001266241074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,12288,0.05936000123620033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,10240,0.05097600072622299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,8192,0.04303999990224838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,16384,0.07577600330114365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,4096,0.025855999439954758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,3072,0.022175999358296394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,3584,0.02380800060927868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,5120,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,2560,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,1536,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,1024,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,512,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,2048,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,256,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,128,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,65536,0.35280001163482666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,12288,0.05190400034189224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,10240,0.043616000562906265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,8192,0.03747199848294258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,16384,0.06572800129652023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,7168,0.03574400022625923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,5120,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,4096,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,3584,0.02191999927163124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,3072,0.022272000089287758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,65536,0.2667199969291687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,2048,0.01692800037562847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,1536,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,2560,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,512,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,1024,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,256,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,128,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,12288,0.043007999658584595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,10240,0.03788800165057182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,7168,0.028896000236272812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,8192,0.030719999223947525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,16384,0.0544000007212162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,5120,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,4096,0.020191999152302742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,3072,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,3584,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,2560,0.018239999189972878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,65536,0.22735999524593353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,2048,0.015231999568641186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,256,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,512,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,1024,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,1536,0.01283199992030859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,128,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,12288,0.03984000161290169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,8192,0.02921600081026554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,10240,0.03478400036692619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,7168,0.02598400041460991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,16384,0.05004800111055374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,5120,0.02127999998629093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,3584,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,3072,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,4096,0.02067199908196926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,65536,0.18572799861431122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,2560,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,2048,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,1024,0.011392000131309032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,1536,0.01398400031030178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,256,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,128,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,512,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,12288,0.03331200033426285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,8192,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,10240,0.028896000236272812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,7168,0.027264000847935677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,16384,0.04156799986958504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,4096,0.017952000722289085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,3584,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,5120,0.020447999238967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,3072,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,2560,0.017983999103307724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,65536,0.16784000396728516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,1024,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,1536,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,2048,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,128,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,12288,0.031168000772595406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,7168,0.02598400041460991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,8192,0.034015998244285583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,10240,0.026847999542951584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,16384,0.039103999733924866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,5120,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,3072,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,4096,0.017920000478625298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,65536,0.1382399946451187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,3584,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,2560,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,2048,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,1536,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,512,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,1024,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,256,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,12288,0.029343999922275543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,8192,0.026016000658273697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,10240,0.026208000257611275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,7168,0.0306560005992651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,16384,0.03705599904060364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,4096,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,3584,0.016736000776290894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,5120,0.02425600029528141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,65536,0.12591999769210815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,3072,0.017696000635623932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,2560,0.014688000082969666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,1536,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,2048,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,512,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,128,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,12288,0.031007999554276466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,10240,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,7168,0.02457600086927414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,8192,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,16384,0.03683200106024742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,65536,0.12252800166606903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,5120,0.02332800067961216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,3584,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,4096,0.01740800030529499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,3072,0.018144000321626663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,2560,0.016448000445961952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,2048,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,1536,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,1024,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,512,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,128,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,12288,0.0306560005992651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,10240,0.029823999851942062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,7168,0.02393599972128868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,8192,0.025472000241279602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,16384,0.035071998834609985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,5120,0.024000000208616257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,4096,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,65536,0.11785600334405899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,3584,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,3072,0.01759999990463257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,2048,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,1024,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,1536,0.013439999893307686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,256,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,128,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,12288,0.035232000052928925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,10240,0.03852799907326698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,8192,0.03177599981427193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,7168,0.022911999374628067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,16384,0.03468799963593483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,5120,0.023615999147295952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,65536,0.11388800293207169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,4096,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,3584,0.019200000911951065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,2560,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,3072,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,2048,0.014495999552309513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,1536,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,512,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,1024,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,256,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,128,0.0077760000713169575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,12288,0.0326399989426136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,10240,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,8192,0.03206399828195572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,16384,0.054816000163555145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,7168,0.029664000496268272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,65536,0.11276800185441971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,3584,0.018464000895619392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,4096,0.02022399939596653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,5120,0.02287999913096428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,3072,0.017791999503970146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,2048,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,2560,0.015359999611973763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,256,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,512,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,1024,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,1536,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,12288,0.044576000422239304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,8192,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,10240,0.029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,7168,0.029759999364614487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,16384,0.05455999821424484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,65536,0.11180800199508667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,5120,0.018880000337958336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,4096,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,3584,0.01836800016462803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,3072,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,2560,0.015744000673294067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,2048,0.012608000077307224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,512,0.009631999768316746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,1536,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,256,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,128,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,12288,0.03286400064826012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,8192,0.024064000695943832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,10240,0.03827200084924698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,7168,0.02284800074994564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,16384,0.054976001381874084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,65536,0.11148799955844879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,4096,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,3072,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,3584,0.018688000738620758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,5120,0.02191999927163124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,2560,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,1024,0.01104000024497509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,512,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,1536,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,2048,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,256,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,128,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,12288,0.04416000097990036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,8192,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,7168,0.02924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,10240,0.03743999823927879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,65536,0.111455999314785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,16384,0.05523199960589409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,5120,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,3584,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,4096,0.020416000857949257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,2560,0.015296000055968761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,2048,0.014431999996304512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,3072,0.017023999243974686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,1536,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,1024,0.010975999757647514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,256,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,512,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,128,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,12288,0.032416000962257385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,7168,0.021888000890612602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,8192,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,10240,0.02848000079393387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,16384,0.04064000025391579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,65536,0.1387840062379837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,5120,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,3072,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,3584,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,4096,0.020160000771284103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,2560,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,2048,0.01360000018030405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,1024,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,1536,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,512,0.009440000168979168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,256,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,128,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,65536,0.13945600390434265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,5120,0.11574400216341019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,7168,0.1554879993200302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,8192,0.17574399709701538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,4096,0.09340800344944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,10240,0.21731199324131012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,12288,0.2603519856929779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,3584,0.08233600109815598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,3072,0.0777600035071373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,2048,0.055615998804569244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,2560,0.0724480003118515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,512,0.04310400038957596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,1536,0.0490880012512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,1024,0.045343998819589615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,256,0.04265600070357323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,128,0.04230400174856186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,16384,0.3426240086555481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,12288,0.07142399996519089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,10240,0.060736000537872314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,8192,0.05087999999523163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,5120,0.03580800071358681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,7168,0.04566400125622749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,16384,0.09212800115346909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,3584,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,4096,0.029632000252604485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,3072,0.026623999699950218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,2560,0.024224000051617622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,2048,0.019392000511288643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,1536,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,256,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,512,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,1024,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,128,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,8192,0.040511999279260635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,12288,0.05689600110054016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,7168,0.03622400015592575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,10240,0.048928000032901764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,16384,0.07318399846553802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,5120,0.029152000322937965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,3072,0.021247999742627144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,4096,0.025472000241279602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,3584,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,2560,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,1536,0.015263999812304974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,2048,0.0163199994713068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,1024,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,512,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,256,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,128,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,12288,0.04870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,65536,0.37830400466918945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,10240,0.041280001401901245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,8192,0.0352960005402565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,16384,0.06188800185918808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,5120,0.025567999109625816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,7168,0.031488001346588135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,4096,0.022655999287962914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,3584,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,3072,0.018688000738620758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,2560,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,2048,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,512,0.012543999589979649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,1024,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,1536,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,65536,0.2984960079193115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,256,0.01244799979031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,128,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,12288,0.040800001472234726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,10240,0.0350399985909462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,8192,0.029152000322937965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,7168,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,16384,0.05056000128388405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,5120,0.021727999672293663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,3072,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,3584,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,2560,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,4096,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,65536,0.25276800990104675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,2048,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,128,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,512,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,1024,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,1536,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,256,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,12288,0.03673600032925606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,8192,0.02672000043094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,7168,0.025728000327944756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,10240,0.03126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,16384,0.04758400097489357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,5120,0.01990400068461895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,3072,0.015104000456631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,4096,0.017791999503970146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,3584,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,65536,0.19881600141525269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,2560,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,2048,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,1024,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,1536,0.012032000347971916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,256,0.01065600011497736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,128,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,512,0.01056000031530857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,8192,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,12288,0.02844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,7168,0.02038400061428547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,10240,0.02486399933695793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,16384,0.03657599911093712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,4096,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,5120,0.01679999940097332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,3072,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,3584,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,65536,0.18835200369358063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,2560,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,1536,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,1024,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,2048,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,512,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,256,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,128,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,12288,0.023135999217629433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,7168,0.016543999314308167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,8192,0.017855999991297722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,10240,0.0208320003002882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,16384,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,65536,0.14243200421333313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,5120,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,4096,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,2560,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,3584,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,3072,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,2048,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,1536,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,1024,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,512,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,128,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,256,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,12288,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,10240,0.01945599913597107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,8192,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,7168,0.016383999958634377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,16384,0.026079999282956123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,65536,0.10025600343942642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,5120,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,4096,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,2560,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,3584,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,3072,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,2048,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,1536,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,1024,0.008448000065982342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,512,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,256,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,128,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,12288,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,10240,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,16384,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,7168,0.014879999682307243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,8192,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,65536,0.09136000275611877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,5120,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,4096,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,3584,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,2560,0.010528000071644783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,3072,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,1536,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,2048,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,1024,0.008128000423312187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,512,0.008128000423312187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,256,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,12288,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,10240,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,7168,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,8192,0.01500799972563982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,65536,0.08630400151014328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,16384,0.023584000766277313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,4096,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,5120,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,2560,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,2048,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,3584,0.010495999827980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,3072,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,1024,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,1536,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,512,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,256,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,128,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,12288,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,8192,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,10240,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,7168,0.011168000288307667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,16384,0.017376000061631203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,65536,0.08476799726486206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,5120,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,3584,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,4096,0.009920000098645687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,2048,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,2560,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,3072,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,512,0.006943999789655209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,1024,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,1536,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,256,0.007135999854654074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,128,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,12288,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,10240,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,8192,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,7168,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,16384,0.014720000326633453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,65536,0.052799999713897705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,5120,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,3584,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,4096,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,3072,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,2560,0.007968000136315823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,1536,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,2048,0.007648000027984381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,1024,0.0071680000983178616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,512,0.006912000011652708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,256,0.006912000011652708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,128,0.006719999946653843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,12288,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,7168,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,8192,0.009472000412642956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,10240,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,16384,0.012736000120639801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,65536,0.04259200021624565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,5120,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,4096,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,3584,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,3072,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,2560,0.007615999784320593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,2048,0.0071680000983178616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,1536,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,1024,0.006591999903321266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,256,0.006624000146985054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,512,0.006719999946653843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,128,0.0066559999249875546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,12288,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,8192,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,10240,0.009375999681651592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,16384,0.01190400030463934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,7168,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,65536,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,5120,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,4096,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,3072,0.0074880002066493034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,3584,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,2048,0.006624000146985054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,2560,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,1536,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,512,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,1024,0.006432000081986189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,256,0.0063680000603199005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,128,0.006496000103652477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,12288,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,7168,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,8192,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,10240,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,16384,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,65536,0.028031999245285988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,4096,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,5120,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,2560,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,3584,0.007552000228315592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,3072,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,2048,0.006783999968320131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,1536,0.00684799998998642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,1024,0.006688000168651342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,512,0.006271999794989824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,128,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,256,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,12288,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,7168,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,10240,0.00886400043964386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,8192,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,16384,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,65536,0.027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,5120,0.008287999778985977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,3584,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,3072,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,2560,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,4096,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,2048,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,1536,0.006719999946653843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,256,0.006496000103652477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,1024,0.006432000081986189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,512,0.006304000038653612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,128,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,65536,0.02703999914228916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,5120,0.11462400108575821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,7168,0.15459200739860535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,8192,0.1746560037136078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,4096,0.09177599847316742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,10240,0.21583999693393707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,12288,0.25734400749206543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,3584,0.08207999914884567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,3072,0.07414399832487106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,2560,0.07081600278615952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,1536,0.04604800045490265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,2048,0.05347200110554695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,512,0.037567999213933945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,1024,0.04057599976658821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,128,0.03625600039958954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,256,0.03667199984192848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,16384,0.340256005525589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,12288,0.07014399766921997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,10240,0.06054399907588959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,7168,0.04521600157022476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,8192,0.049536000937223434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,5120,0.03478400036692619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,16384,0.0907519981265068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,2560,0.02316799946129322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,4096,0.028224000707268715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,3072,0.025248000398278236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,3584,0.027103999629616737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,2048,0.019231999292969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,128,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,256,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,1024,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,1536,0.016416000202298164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,512,0.014015999622642994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,7168,0.035392001271247864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,8192,0.040800001472234726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,12288,0.05471999943256378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,10240,0.046911999583244324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,16384,0.07072000205516815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,3072,0.019936000928282738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,3584,0.021376000717282295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,5120,0.02816000021994114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,4096,0.023840000852942467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,2560,0.0197759997099638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,2048,0.01583999954164028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,1536,0.014112000353634357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,1024,0.012959999963641167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,512,0.01235199999064207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,256,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,128,0.01206399966031313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,65536,0.34191998839378357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,12288,0.047520000487565994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,10240,0.04028800129890442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,8192,0.03404799848794937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,7168,0.031647998839616776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,16384,0.060256000608205795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,3584,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,4096,0.020767999812960625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,5120,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,3072,0.018303999677300453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,2560,0.01696000061929226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,65536,0.2654399871826172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,2048,0.013952000066637993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,1536,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,512,0.011487999930977821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,256,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,1024,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,128,0.011327999643981457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,12288,0.038975998759269714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,8192,0.029023999348282814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,10240,0.03308799862861633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,7168,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,16384,0.049215998500585556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,5120,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,4096,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,3584,0.016863999888300896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,3072,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,2560,0.014783999882638454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,65536,0.2197120040655136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,2048,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,1024,0.010591999627649784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,256,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,1536,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,512,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,128,0.009983999654650688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,12288,0.03638400137424469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,7168,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,10240,0.031328000128269196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,8192,0.02675200067460537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,16384,0.04492799937725067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,5120,0.019807999953627586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,4096,0.01648000068962574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,3072,0.014495999552309513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,3584,0.015456000342965126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,65536,0.1777919977903366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,2560,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,2048,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,1536,0.010944000445306301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,256,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,128,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,512,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,1024,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,12288,0.02812799997627735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,7168,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,8192,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,10240,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,16384,0.035679999738931656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,5120,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,65536,0.16495999693870544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,3072,0.012671999633312225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,3584,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,4096,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,2560,0.011807999573647976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,2048,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,1536,0.0098879998549819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,1024,0.009151999838650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,128,0.008671999908983707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,512,0.008832000195980072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,256,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,12288,0.02319999970495701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,10240,0.020031999796628952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,7168,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,8192,0.01744000054895878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,16384,0.02816000021994114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,65536,0.11984000355005264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,5120,0.013567999936640263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,4096,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,2560,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,3584,0.011359999887645245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,3072,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,2048,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,512,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,1024,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,1536,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,256,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,128,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,12288,0.02131200022995472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,10240,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,8192,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,7168,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,16384,0.025919999927282333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,65536,0.09267199784517288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,5120,0.012703999876976013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,4096,0.011455999687314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,2560,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,3584,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,3072,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,2048,0.009184000082314014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,1536,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,512,0.007679999805986881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,1024,0.008415999822318554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,256,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,128,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,12288,0.019360000267624855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,10240,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,8192,0.015039999969303608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,16384,0.02409599907696247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,7168,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,65536,0.08291199803352356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,5120,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,4096,0.011552000418305397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,3072,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,2560,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,3584,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,2048,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,1536,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,1024,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,512,0.007424000184983015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,128,0.007679999805986881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,256,0.007679999805986881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,12288,0.017791999503970146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,10240,0.016448000445961952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,16384,0.021824000403285027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,8192,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,7168,0.013791999779641628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,65536,0.07500799745321274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,5120,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,3584,0.010143999941647053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,4096,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,2048,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,3072,0.009600000455975533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,2560,0.00940799992531538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,1536,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,1024,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,512,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,256,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,128,0.007615999784320593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,12288,0.014751999638974667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,10240,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,8192,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,16384,0.01727999933063984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,7168,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,65536,0.07056000083684921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,5120,0.010879999957978725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,3584,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,4096,0.008991999551653862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,2048,0.007679999805986881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,3072,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,2560,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,1536,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,512,0.006816000211983919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,1024,0.007071999832987785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,256,0.006912000011652708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,128,0.006688000168651342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,12288,0.012480000033974648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,10240,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,8192,0.010367999784648418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,7168,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,16384,0.014816000126302242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,5120,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,65536,0.05011200159788132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,2560,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,4096,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,3584,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,3072,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,2048,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,512,0.0066559999249875546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,1024,0.00687999976798892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,128,0.006688000168651342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,1536,0.00687999976798892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,256,0.006912000011652708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,12288,0.011008000001311302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,10240,0.010432000271975994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,8192,0.009247999638319016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,7168,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,16384,0.012512000277638435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,65536,0.04070400074124336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,5120,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,4096,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,3072,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,3584,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,2560,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,1536,0.0066559999249875546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,2048,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,1024,0.006560000125318766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,512,0.006688000168651342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,128,0.007040000054985285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,256,0.006560000125318766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,12288,0.009344000369310379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,10240,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,8192,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,16384,0.010463999584317207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,7168,0.008224000222980976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,65536,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,5120,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,3584,0.006976000033318996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,4096,0.0071680000983178616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,3072,0.007040000054985285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,2048,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,2560,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,1536,0.006399999838322401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,512,0.006432000081986189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,1024,0.006432000081986189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,128,0.006240000016987324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,256,0.006591999903321266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,12288,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,10240,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,8192,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,16384,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,7168,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,65536,0.024288000538945198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,5120,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,4096,0.006783999968320131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,3584,0.007071999832987785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,3072,0.007071999832987785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,2560,0.007007999811321497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,2048,0.006719999946653843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,1536,0.006496000103652477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,1024,0.006527999881654978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,512,0.006304000038653612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,256,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,128,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,12288,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,8192,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,10240,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,16384,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,7168,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,65536,0.022112000733613968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,5120,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,3584,0.00684799998998642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,4096,0.00684799998998642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,3072,0.007007999811321497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,2560,0.006976000033318996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,2048,0.006175999995321035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,1536,0.006304000038653612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,256,0.006432000081986189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,512,0.006175999995321035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,1024,0.006240000016987324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,128,0.006527999881654978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,65536,0.02115200087428093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,5120,0.11139199882745743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,7168,0.15347200632095337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,4096,0.0902400016784668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,8192,0.1719679981470108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,10240,0.21484799683094025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,12288,0.25593599677085876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,2560,0.06188800185918808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,2048,0.04816000163555145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,3584,0.08032000064849854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,3072,0.07091200351715088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,1536,0.04243199899792671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,1024,0.03308799862861633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,512,0.028575999662280083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,256,0.02828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,128,0.028224000707268715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,16384,0.3383359909057617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,12288,0.06988800317049026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,10240,0.059328000992536545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,8192,0.04899200052022934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,7168,0.04403200000524521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,5120,0.03363199904561043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,16384,0.09081599861383438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,4096,0.02783999964594841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,3584,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,3072,0.023360000923275948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,2560,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,1536,0.01587199978530407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,2048,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,512,0.011872000060975552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,1024,0.013311999849975109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,256,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,128,0.012000000104308128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,12288,0.05363199859857559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,8192,0.03836800158023834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,7168,0.03433600068092346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,10240,0.04560000076889992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,16384,0.06908799707889557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,4096,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,5120,0.0272000003606081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,3072,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,3584,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,2560,0.017503999173641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,2048,0.01484800036996603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,1536,0.013055999763309956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,1024,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,512,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,128,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,256,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,65536,0.34515199065208435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,12288,0.045823998749256134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,8192,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,10240,0.04032000154256821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,7168,0.031039999797940254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,16384,0.06047999858856201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,5120,0.023615999147295952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,2560,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,3072,0.016992000862956047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,3584,0.01849599927663803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,4096,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,65536,0.25913599133491516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,512,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,2048,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,1536,0.012384000234305859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,1024,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,128,0.010208000428974628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,256,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,12288,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,8192,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,10240,0.03251200169324875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,7168,0.025312000885605812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,16384,0.04806400090456009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,5120,0.020287999883294106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,4096,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,3584,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,3072,0.014976000413298607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,65536,0.21961599588394165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,2560,0.014047999866306782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,1536,0.011648000217974186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,2048,0.011935999616980553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,1024,0.009952000342309475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,256,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,512,0.008895999751985073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,128,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,12288,0.03545600175857544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,10240,0.030208000913262367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,8192,0.025696000084280968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,7168,0.023072000592947006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,16384,0.04476799815893173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,5120,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,4096,0.016287999227643013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,3584,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,3072,0.013919999822974205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,2560,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,65536,0.17683200538158417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,2048,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,1536,0.011136000044643879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,256,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,128,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,512,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,1024,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,12288,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,10240,0.022943999618291855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,16384,0.033344000577926636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,7168,0.018271999433636665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,8192,0.020959999412298203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,5120,0.015039999969303608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,4096,0.013407999649643898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,3584,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,65536,0.1565759927034378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,2560,0.01119999960064888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,3072,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,2048,0.010080000385642052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,512,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,1536,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,1024,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,256,0.007968000136315823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,128,0.00800000037997961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,12288,0.02271999977529049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,10240,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,7168,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,8192,0.01724799908697605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,16384,0.02816000021994114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,5120,0.013472000136971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,65536,0.11206399649381638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,4096,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,2560,0.01033599954098463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,3584,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,3072,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,1536,0.00848000030964613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,2048,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,1024,0.008191999979317188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,512,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,128,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,256,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,12288,0.021344000473618507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,10240,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,16384,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,7168,0.015936000272631645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,8192,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,5120,0.013120000250637531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,65536,0.0939520001411438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,4096,0.011839999817311764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,3584,0.01152000017464161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,2048,0.009119999594986439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,3072,0.010400000028312206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,2560,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,1024,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,512,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,1536,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,256,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,128,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,12288,0.01820800080895424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,8192,0.0144640002399683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,10240,0.01603199914097786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,7168,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,16384,0.0225600004196167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,65536,0.08640000224113464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,5120,0.01142400037497282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,4096,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,3584,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,2560,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,3072,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,2048,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,512,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,1024,0.007648000027984381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,1536,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,128,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,256,0.007071999832987785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,12288,0.01635199971497059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,10240,0.014527999795973301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,7168,0.012160000391304493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,8192,0.013024000450968742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,16384,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,5120,0.010623999871313572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,4096,0.009727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,65536,0.07097599655389786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,3584,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,2560,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,3072,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,2048,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,1536,0.008383999578654766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,1024,0.007071999832987785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,512,0.007007999811321497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,256,0.006912000011652708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,128,0.00687999976798892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,8192,0.01158399973064661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,12288,0.014399999752640724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,10240,0.013088000006973743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,7168,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,16384,0.017216000705957413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,65536,0.05987200140953064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,5120,0.009664000011980534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,3584,0.008704000152647495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,4096,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,2560,0.008031999692320824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,3072,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,2048,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,1024,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,1536,0.0072639998979866505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,256,0.006783999968320131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,512,0.006752000190317631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,128,0.006688000168651342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,12288,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,10240,0.011264000087976456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,8192,0.011103999800980091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,7168,0.01017600018531084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,16384,0.014368000440299511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,65536,0.05119999870657921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,5120,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,4096,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,3584,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,2560,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,2048,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,3072,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,1536,0.00684799998998642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,1024,0.006560000125318766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,128,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,256,0.006688000168651342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,512,0.006624000146985054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,12288,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,8192,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,10240,0.010111999697983265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,16384,0.012319999746978283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,7168,0.008927999995648861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,65536,0.04022400081157684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,5120,0.0081599997356534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,4096,0.007424000184983015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,3584,0.007519999984651804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,3072,0.007615999784320593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,2560,0.007679999805986881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,2048,0.007040000054985285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,1536,0.006624000146985054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,512,0.006432000081986189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,256,0.006432000081986189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,1024,0.0066559999249875546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,128,0.006560000125318766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,12288,0.009503999724984169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,8192,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,10240,0.00863999966531992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,7168,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,16384,0.010847999714314938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,65536,0.03017600066959858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,5120,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,4096,0.007360000163316727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,3584,0.007104000076651573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,3072,0.007071999832987785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,2560,0.00723200011998415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,1536,0.006463999859988689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,512,0.006304000038653612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,1024,0.006496000103652477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,2048,0.006719999946653843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,128,0.006207999773323536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,256,0.0063680000603199005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,12288,0.008960000239312649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,8192,0.007903999648988247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,7168,0.007935999892652035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,10240,0.008736000396311283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,16384,0.009696000255644321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,65536,0.023584000766277313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,5120,0.007327999919652939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,2048,0.006304000038653612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,4096,0.006976000033318996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,3584,0.007455999962985516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,3072,0.006912000011652708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,2560,0.007199999876320362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,1536,0.006496000103652477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,512,0.006399999838322401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,1024,0.006335999816656113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,256,0.006912000011652708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,128,0.006399999838322401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,12288,0.008511999621987343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,65536,0.02195199951529503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,16384,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,10240,0.00825599953532219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,8192,0.00774399982765317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,7168,0.007872000336647034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,4096,0.007007999811321497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,5120,0.007296000141650438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,3584,0.00687999976798892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,3072,0.0066559999249875546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,2560,0.006912000011652708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,2048,0.006175999995321035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,1536,0.006271999794989824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,512,0.006271999794989824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,1024,0.006240000016987324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,256,0.00598399993032217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,128,0.006207999773323536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,65536,0.02099199965596199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,5120,7.591921997070313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,6144,9.110249837239582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,7168,10.372593180338542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,8192,11.824148559570313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,10240,14.727747599283854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,3584,5.289503987630209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,3072,4.435957336425782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,4096,6.073353576660156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,2560,3.6780372619628907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,512,0.8420501073201498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,768,1.162135442097982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,1024,1.5353333791097006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,2048,3.000249481201172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,256,0.5270058631896972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,128,0.4421824137369792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,64,0.41065066655476884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,1536,2.309866587320964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,32,0.3996469179789225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,12288,17.80384724934896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,16384,25.25881551106771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,8192,2.8652885437011717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,7168,2.5970133463541667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,10240,3.808818054199219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,12288,4.405024210611979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,6144,2.1929588317871094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,5120,1.8048576354980468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,4096,1.4033611297607422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,3584,1.2888671875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,3072,1.0544352213541666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,16384,6.02752939860026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,2560,0.8827893575032553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,1024,0.38728319803873695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,1536,0.5473397572835286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,2048,0.7074527740478516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,256,0.13121706644694012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,512,0.21833066940307616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,768,0.2972074508666992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,64,0.1078218698501587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,128,0.11478079954783123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,32,0.10361920197804768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,7168,1.8198666890462238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,8192,2.0994997660319012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,10240,2.6978485107421877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,12288,3.1564565022786457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,6144,1.516857655843099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,5120,1.2939093271891275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,4096,1.037396240234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,3584,0.9003018697102865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,16384,4.26709238688151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,1024,0.2791296005249023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,3072,0.7662314732869466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,2048,0.5266175905863444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,1536,0.4014303843180339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,2560,0.6642293294270833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,256,0.09896639982859293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,768,0.21870719591776527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,128,0.08719253540039062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,512,0.15930026372273762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,64,0.0799295981725057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,32,0.07885653177897135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,8192,1.7334388732910155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,10240,2.269819641113281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,12288,2.641828155517578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,16384,3.624476877848307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,7168,1.5582112630208333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,6144,1.3118228912353516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,5120,1.061828295389811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,4096,0.8669696172078452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,3584,0.7649269104003906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,2560,0.5455381393432617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,3072,0.6690282821655273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,1024,0.2354207992553711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,768,0.1913472016652425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,1536,0.34126612345377605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,2048,0.44525760014851884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,128,0.07486399809519449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,256,0.09363839626312256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,512,0.13803733189900716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,64,0.06912960211435953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,32,0.06897386709849039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,65536,25.393603515625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,10240,1.7727476755777996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,12288,2.0648117065429688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,16384,2.836747741699219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,7168,1.2339861551920572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,8192,1.4211519877115886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,6144,1.054033088684082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,5120,0.8626218795776367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,4096,0.7029397328694661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,3584,0.6135114669799805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,3072,0.5324554761250814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,65536,17.647281901041666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,2560,0.44313494364420575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,1024,0.1921855926513672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,2048,0.3636586825052897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,768,0.15080320040384929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,1536,0.27379945119222004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,128,0.06005226771036783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,256,0.06795519987742106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,64,0.054819198449452725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,512,0.10821333726247151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,32,0.05418453216552734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,8192,1.1862698872884114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,10240,1.4827541351318358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,12288,1.7943487803141278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,65536,15.021636962890625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,16384,2.4917706807454425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,7168,1.0383541107177734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,6144,0.884880002339681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,5120,0.7397450764973958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,3072,0.4496010780334473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,3584,0.5265717188517253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,4096,0.6002143859863281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,2560,0.3757802645365397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,2048,0.3058527946472168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,256,0.06069653431574503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,512,0.09573866526285807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,768,0.1299605369567871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,1024,0.16409707069396973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,1536,0.23237973848978677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,128,0.05412906805674235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,64,0.04928640127182007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,32,0.047787733872731525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,8192,1.0490026473999023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,10240,1.3118666330973308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,12288,1.5464181264241537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,16384,2.080199432373047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,65536,12.5071533203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,7168,0.907319450378418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,6144,0.7714154561360677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,5120,0.6492959976196289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,3072,0.3988032023111979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,3584,0.47403628031412764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,2560,0.33204800287882485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,2048,0.2702399889628092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,4096,0.5214005470275879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,1024,0.14822079340616862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,256,0.05341653426488241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,512,0.08337706724802653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,768,0.11139946778615314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,1536,0.20737600326538086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,128,0.04791040023167928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,64,0.04344746669133504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,32,0.0423093318939209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,8192,0.8507285435994467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,10240,1.055623435974121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,12288,1.2944117228190104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,16384,1.692422358194987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,7168,0.7463071823120118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,6144,0.6380842844645183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,5120,0.5503733317057292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,65536,10.366147867838542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,4096,0.43543361028035477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,3584,0.3809237480163574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,2048,0.23173012733459472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,3072,0.3289802551269531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,2560,0.2849952061971029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,512,0.07162666320800781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,256,0.04868693351745605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,768,0.09666879971822104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,1024,0.12118186950683593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,1536,0.17335039774576824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,128,0.039892268180847165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,64,0.036149334907531736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,32,0.03588800032933553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,8192,0.6754965464274089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,10240,0.8551968256632486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,12288,1.0003637313842773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,16384,1.3403221130371095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,7168,0.5938592274983724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,6144,0.5193322817484538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,65536,9.012565104166667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,5120,0.42281068166097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,3584,0.30369707743326824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,2048,0.17589546839396158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,3072,0.2668959935506185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,4096,0.34351358413696287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,2560,0.22192106246948243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,256,0.03769493500391642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,768,0.07795200347900391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,512,0.05664746761322022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,1024,0.09539946715037027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,1536,0.13684479395548504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,64,0.029597866535186767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,128,0.033114665746688844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,32,0.03011946678161621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,8192,0.5896213531494141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,10240,0.7352383931477864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,12288,0.8971839904785156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,16384,1.1861610412597656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,7168,0.5179189364115397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,65536,7.293996683756511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,6144,0.44674132664998367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,5120,0.38226985931396484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,2560,0.1932949384053548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,4096,0.3034229278564453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,3584,0.26591466267903646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,3072,0.22935360272725425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,2048,0.15976959864298504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,512,0.05052266518274943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,256,0.033701332410176595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,768,0.06745920181274415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,1536,0.12021013100941975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,1024,0.08528746763865153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,64,0.026786132653554277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,128,0.029383466641108198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,32,0.026881066958109538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,8192,0.5079637209574381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,10240,0.6271157582600911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,12288,0.7731007893880208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,16384,1.0182506561279296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,65536,5.855290730794271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,7168,0.4466890652974446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,6144,0.38089173634847007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,5120,0.33007040023803713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,4096,0.2622005303700765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,3072,0.1995968023935954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,2048,0.13840319315592448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,3584,0.2309055964152018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,2560,0.16556159655253094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,256,0.030180267492930096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,768,0.06033493280410766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,512,0.04459733168284098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,1024,0.07463786602020264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,1536,0.10507733027140301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,128,0.02590720057487488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,64,0.02391466697057088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,32,0.024871466557184856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,8192,0.42079893747965497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,10240,0.5377525329589844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,12288,0.6219722747802734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,16384,0.8382730484008789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,6144,0.3271114667256673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,7168,0.36985387802124026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,65536,4.977044169108073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,5120,0.26527679761250816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,3072,0.1654047966003418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,2560,0.14007253646850587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,4096,0.21776213645935058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,2048,0.11310186386108398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,3584,0.19550293286641438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,256,0.027668267488479614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,512,0.038466131687164305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,768,0.05080960194269816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,1024,0.0626421332359314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,1536,0.08824426333109538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,128,0.022012799978256226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,64,0.019526400168736777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,32,0.0202239990234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,8192,0.3378005345662435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,10240,0.4133066813151042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,12288,0.5111754735310872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,7168,0.29681708017985026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,16384,0.661192512512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,6144,0.25157334009806315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,65536,4.350884501139323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,5120,0.21843412717183433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,2560,0.11283093293507893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,3584,0.15238505999247234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,3072,0.13064320087432862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,4096,0.17369813919067384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,2048,0.09327999750773111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,256,0.021961599588394165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,768,0.0405290683110555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,512,0.031129600604375203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,1024,0.0505791982014974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,1536,0.070796799659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,128,0.016884267330169678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,64,0.01474133332570394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,32,0.015337600310643514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,8192,0.2540991942087809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,10240,0.31133759816487633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,12288,0.3844362576802572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,16384,0.4984554608662923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,7168,0.223418664932251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,65536,3.4459444681803384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,6144,0.19001065889994304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,5120,0.16527786254882812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,2560,0.08415466944376628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,3584,0.11595839659372967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,3072,0.10095679759979248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,4096,0.13109440008799236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,2048,0.07112853527069092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,768,0.03248959978421529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,512,0.025085866451263428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,1024,0.0394165317217509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,256,0.0177130659421285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,1536,0.05385706822077433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,64,0.012157866358757019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,32,0.01234346628189087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,128,0.013839999834696451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,8192,0.16919573148091632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,10240,0.21066133181254068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,12288,0.2505781332651774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,16384,0.3285248120625814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,7168,0.14879892667134603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,6144,0.12876053651173908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,65536,2.788439432779948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,5120,0.1078986644744873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,2560,0.0572703997294108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,3072,0.06738026936848959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,3584,0.07731839815775553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,4096,0.08636906941731771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,2048,0.04666026830673218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,1024,0.0257205327351888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,256,0.011348266402880352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,512,0.01585919956366221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,768,0.021220266819000244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,128,0.008740267157554627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,1536,0.035877335071563723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,64,0.007673599819342296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,32,0.008063999811808269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,8192,0.1306549310684204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,10240,0.16426026026407878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,12288,0.19416853586832683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,16384,0.26127146085103353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,65536,2.012939707438151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,7168,0.11483519872029621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,6144,0.10027306874593098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,5120,0.08349760373433432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,4096,0.06869440078735352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,3584,0.06055999994277954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,3072,0.0521237333615621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,2048,0.036187732219696046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,2560,0.044599465529123944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,1536,0.028467200199762982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,256,0.009040000041325887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,768,0.016085333625475564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,512,0.012727466225624085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,1024,0.020501333475112914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,128,0.0072618668278058365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,64,0.006330666442712148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,32,0.0065077334642410275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,8192,0.08730879624684652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,10240,0.10923306941986084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,12288,0.13075520197550455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,16384,0.17119253476460775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,65536,1.358465067545573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,7168,0.07691946824391684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,6144,0.06750293572743735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,5120,0.0570090651512146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,3584,0.04193813403447469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,2048,0.024734934171040855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,3072,0.03549439907073974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,4096,0.04591466585795085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,2560,0.03071039915084839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,256,0.007276799778143566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,512,0.00973653296629588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,1024,0.014045866330464682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,768,0.011703466375668842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,1536,0.019386667013168334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,32,0.005696000158786773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,128,0.006163200239340464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,64,0.005639466643333435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,8192,0.053445335229237875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,10240,0.06620053450266519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,12288,0.07890453338623046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,65536,1.0321706771850585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,16384,0.10313386917114258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,7168,0.04730986754099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,6144,0.04234346548716227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,5120,0.03595306475957234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,4096,0.029986133178075153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,3584,0.027036799987157183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,3072,0.023884799083073935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,2048,0.01405333379904429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,2560,0.020716800292332967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,256,0.0052042668064435325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,1024,0.008538666367530822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,512,0.006177066763242086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,1536,0.011266133189201355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,768,0.007394133508205414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,128,0.004542933404445648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,32,0.004287999868392944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,64,0.004107733319203059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,8192,0.04867839813232422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,10240,0.06026560068130493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,12288,0.07133013407389323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,65536,0.6706560134887696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,16384,0.10279893080393473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,7168,0.04312959909439087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,6144,0.03847786585489909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,5120,0.03249280055363973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,4096,0.026770132780075073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,3584,0.023970133066177367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,3072,0.021451733509699502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,2560,0.017032533884048462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,2048,0.009453866879145305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,512,0.0047199999292691554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,1024,0.006040533383687338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,256,0.004078933348258337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,768,0.005490133166313171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,1536,0.007523199915885926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,128,0.0036202666660149893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,32,0.0036458666125933326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,64,0.0033632000287373864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,8192,0.04730666478474935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,10240,0.0582698663075765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,12288,0.06940906842549642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,65536,0.40601387023925783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,16384,0.09027840296427408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,7168,0.04193386634190877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,6144,0.037390931447347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,5120,0.03145280083020528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,4096,0.026199465990066527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,3584,0.02319999933242798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,2048,0.009474133451779682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,3072,0.0204693337281545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,2560,0.014460800091425577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,1536,0.007959466675917308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,768,0.005393066505591074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,512,0.004538666705290476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,256,0.003849600007136663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,1024,0.00618453323841095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,64,0.0032799998919169106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,128,0.0033802665770053864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,32,0.003357866654793421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,8192,0.047014399369557695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,10240,0.05760746796925863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,12288,0.06851627031962076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,65536,0.38933226267496746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,16384,0.08934400081634522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,7168,0.04143040180206299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,5120,0.03095253308614095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,6144,0.036541867256164554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,4096,0.025740800301233928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,3584,0.02271359960238139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,3072,0.020163200298945107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,2560,0.01385599970817566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,2048,0.008070399860541026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,512,0.0040522667268912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,256,0.003525333354870478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,1536,0.0072053333123524976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,768,0.004867200056711833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,1024,0.005378133555253347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,128,0.003252266595760981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,64,0.0029824001093705496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,32,0.003089066594839096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,65536,0.36568854649861654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,7168,5.144450378417969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,8192,5.888623046875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,10240,7.279030354817708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,6144,4.212907663981119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,65536,0.35958080291748046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,12288,8.890325927734375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,4096,2.8297749837239583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,5120,3.636063893636068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,3584,2.5516586303710938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,1024,0.7393280029296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,3072,2.1971455891927083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,768,0.5844671885172527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,1536,1.0651007970174153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,2048,1.4293930053710937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,128,0.22456107139587403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,2560,1.7777013142903644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,256,0.27348480224609373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,64,0.20961599349975585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,512,0.4256842613220215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,32,0.20307413736979166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,16384,11.97449239095052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,7168,1.207530721028646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,8192,1.4133983612060548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,6144,1.059714126586914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,10240,1.759444300333659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,12288,2.249254353841146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,4096,0.702785046895345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,3584,0.616868273417155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,5120,0.8702880223592123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,3072,0.5290250778198242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,2560,0.459112548828125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,512,0.11108907063802083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,16384,2.9524309794108072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,768,0.1463637351989746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,1024,0.19168747266133626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,1536,0.27740478515625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,256,0.06808213392893472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,2048,0.36430721282958983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,128,0.06766613324483237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,64,0.05529706478118897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,32,0.055949866771698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,7168,0.877566909790039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,8192,0.9975200017293295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,10240,1.2570613861083983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,12288,1.5727179209391275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,5120,0.6305813471476237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,6144,0.7548896153767903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,16384,2.176771291097005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,4096,0.514138666788737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,3584,0.45260480244954426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,1024,0.14129385948181153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,2048,0.27009172439575196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,1536,0.20222506523132325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,2560,0.32436908086140953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,3072,0.3862645467122396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,512,0.0848853349685669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,768,0.11199359893798828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,256,0.052526934941609704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,32,0.04374080101648967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,128,0.046770131587982176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,64,0.043230934937795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,8192,0.8464138666788736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,10240,1.0635829289754233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,12288,1.3134165445963542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,16384,1.7182762145996093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,7168,0.7402762730916341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,6144,0.6381429036458333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,5120,0.5498687744140625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,4096,0.43485654195149737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,3584,0.3791253407796224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,2560,0.2849045435587565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,3072,0.329144541422526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,1024,0.12407893339792889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,1536,0.17178346316019694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,768,0.09614293575286866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,512,0.07331840197245279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,2048,0.22683199246724448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,64,0.03631360133488973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,256,0.04936746756235759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,128,0.03997546831766764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,32,0.036969598134358725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,65536,12.081516520182292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,10240,0.844546127319336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,12288,0.9991914749145507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,8192,0.6815466562906901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,16384,1.3514485677083334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,7168,0.5926293055216472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,6144,0.5054282824198405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,65536,9.064083862304688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,5120,0.4284661293029785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,4096,0.3409205436706543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,3584,0.30318078994750974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,2048,0.17889493306477863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,1536,0.13533867200215657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,3072,0.25952107111612954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,2560,0.22304746309916176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,1024,0.09657920201619466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,768,0.07693866888682047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,256,0.037307735284169516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,128,0.0327349325021108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,512,0.05653866529464722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,32,0.03029973308245341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,64,0.029233066240946452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,8192,0.5902677536010742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,10240,0.7387018839518229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,12288,0.875821876525879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,65536,7.4715423583984375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,16384,1.1703392028808595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,7168,0.5183072090148926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,6144,0.4487637201944987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,4096,0.30496959686279296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,5120,0.37002134323120117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,2560,0.19395626386006673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,3072,0.23008853594462075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,3584,0.26514239311218263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,2048,0.15648533503214518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,1536,0.11893973350524903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,1024,0.08534080187479655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,512,0.05071359872817993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,768,0.06777386665344239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,256,0.03350186745325724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,128,0.029214932521184282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,64,0.02622186740239461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,32,0.03020799954732259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,8192,0.5045525232950847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,10240,0.6347701390584309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,12288,0.748248545328776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,65536,6.064392598470052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,16384,0.9993962605794271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,6144,0.3858751932779948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,7168,0.4431050618489583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,4096,0.2688426653544108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,5120,0.3182421366373698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,3584,0.22951040267944336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,2560,0.1658122698465983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,2048,0.1379637400309245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,1536,0.1031061331431071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,3072,0.19843093554178876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,256,0.02954026659329732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,128,0.025617067019144697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,512,0.044499198595682785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,768,0.058473598957061765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,1024,0.07353813648223877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,32,0.024974934260050454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,64,0.0240064005057017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,8192,0.42263145446777345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,10240,0.5334688186645508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,12288,0.6349002838134765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,16384,0.8447477340698242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,65536,4.924182637532552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,7168,0.3713653246561686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,6144,0.324346669514974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,5120,0.2720383961995443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,2560,0.14205439885457355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,4096,0.21841920216878258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,3072,0.16547733942667645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,3584,0.1959178606669108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,2048,0.11559253533681232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,1536,0.08972266515096029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,512,0.03927786747614543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,1024,0.06340266863505045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,256,0.0278656005859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,128,0.02258560061454773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,768,0.050777598222096765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,32,0.020258132616678873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,64,0.0197760005791982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,8192,0.3343647956848145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,10240,0.4178421338399251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,12288,0.4965685208638509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,65536,4.169300333658854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,16384,0.6703840255737304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,7168,0.2939157485961914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,5120,0.21223146120707193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,3584,0.15380800565083821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,6144,0.2547082742055257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,4096,0.175819730758667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,2048,0.09188799858093262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,1024,0.05073279937108358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,1536,0.07063679695129395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,2560,0.11098453203837078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,3072,0.13154346942901612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,768,0.04098666508992513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,512,0.03151893417040507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,256,0.021824000279108684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,32,0.015539200107256571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,128,0.01686613361040751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,64,0.014823466539382935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,8192,0.31279147466023766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,10240,0.3876554807027181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,12288,0.47069333394368484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,65536,3.587481689453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,16384,0.6205130894978841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,7168,0.27489385604858396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,6144,0.23647467295328775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,4096,0.16764906247456868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,5120,0.20149547259012857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,3584,0.14314026832580568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,3072,0.12320960362752278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,2048,0.08815893332163492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,1536,0.06698666413625082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,2560,0.10421546300252278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,1024,0.048536535104115805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,256,0.01996906598409017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,512,0.029501867294311524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,768,0.039483733971913654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,128,0.015077333648999533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,64,0.014722133676211039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,32,0.015258666872978211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,8192,0.25610666275024413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,10240,0.31613334019978845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,65536,2.7952234903971354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,12288,0.37484585444132484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,16384,0.4982858657836914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,7168,0.22532374064127603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,4096,0.1305578629175822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,5120,0.16397226651509603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,6144,0.19305280049641926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,3072,0.1015328009923299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,3584,0.11554880142211914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,1536,0.054378668467203774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,2048,0.0696618636449178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,2560,0.08471466700236002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,1024,0.03950080076853434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,768,0.032034132877985635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,64,0.011847466230392456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,512,0.02482453385988871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,128,0.013516799608866373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,256,0.018101332585016887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,32,0.012619733810424805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,8192,0.20866559346516927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,10240,0.2625813325246175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,65536,2.6262229919433593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,12288,0.30931520462036133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,16384,0.4128085454305013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,4096,0.10750400225321452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,6144,0.16014293034871419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,5120,0.13494613965352376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,7168,0.1836842695871989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,3584,0.09561813672383626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,2048,0.057860267162322995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,2560,0.0704639991124471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,3072,0.08218773206075033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,1536,0.044570668538411455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,1024,0.03188266754150391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,768,0.025858134031295776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,512,0.020003199577331543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,128,0.01048533320426941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,256,0.013579733173052468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,64,0.00912000040213267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,32,0.00946666697661082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,8192,0.16766613324483234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,10240,0.20714346567789713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,65536,2.1218602498372396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,12288,0.24576106071472167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,16384,0.3267807960510254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,5120,0.10960853099822998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,7168,0.14746774037679036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,4096,0.08562133312225342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,6144,0.12654613653818766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,3072,0.06800320148468017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,1536,0.0356544017791748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,2560,0.056296531359354654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,3584,0.0756330649058024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,2048,0.04581866661707561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,1024,0.025628799200057985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,512,0.01598186691602071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,768,0.020784000555674233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,256,0.011246933539708456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,128,0.008749866485595703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,32,0.008152533570925396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,64,0.007586133480072021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,8192,0.1336085319519043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,65536,1.6840394337972004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,10240,0.16209492683410645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,12288,0.1915071964263916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,16384,0.2543008009592692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,5120,0.08326613108317057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,7168,0.11758933067321778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,6144,0.09978453318277994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,4096,0.0670517365137736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,1536,0.027396267652511595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,2048,0.035906132062276205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,3584,0.05910186767578125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,3072,0.05194133520126343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,2560,0.044712531566619876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,256,0.00899733304977417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,128,0.007283199826876323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,768,0.016170666615168253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,1024,0.019756799936294554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,512,0.012568533420562744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,64,0.006361599763234456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,32,0.006706133484840393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,8192,0.08711679776509604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,10240,0.10558719635009765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,65536,1.3425919850667318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,12288,0.12454506556193035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,16384,0.164408540725708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,7168,0.07671999931335449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,4096,0.04407680034637451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,5120,0.054922668139139805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,6144,0.06510719855626425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,3072,0.03494079907735188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,3584,0.03917866547902425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,1536,0.019283199310302736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,2048,0.024206932385762533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,2560,0.028638933102289838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,1024,0.014146133263905843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,768,0.011699199676513672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,512,0.009603200356165568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,64,0.005421866476535797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,256,0.007267199953397115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,128,0.0060586666067441305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,32,0.005605333546797434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,8192,0.0659114678700765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,10240,0.08189760049184164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,65536,1.0123509089152019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,12288,0.09852159818013509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,16384,0.1326250632603963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,4096,0.03506773312886556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,5120,0.0428607980410258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,6144,0.05039786497751871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,7168,0.0581493337949117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,3584,0.03164479931195577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,1024,0.011241599917411804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,3072,0.026598399877548216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,1536,0.014856533209482829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,2560,0.022808533906936646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,2048,0.018720000982284546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,768,0.009562666217486065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,256,0.006033066908518473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,32,0.004841599861780802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,512,0.00784853349129359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,128,0.005258666475613912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,64,0.00462719996770223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,65536,0.6575093587239583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,8192,0.0460533340771993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,10240,0.05726933479309082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,12288,0.06873706976572672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,16384,0.09315199851989746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,6144,0.035411198933919266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,4096,0.024910932779312132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,5120,0.030138667424519854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,7168,0.0412778655687968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,3584,0.022653865814208984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,1536,0.010829866925875346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,2048,0.013244799772898355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,2560,0.016241066654523215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,3072,0.018649599949518838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,768,0.007336533566315968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,256,0.005163733164469401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,1024,0.008504533767700195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,512,0.006234666705131531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,128,0.004484266539414724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,64,0.004092800120512644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,32,0.004242133100827535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,65536,0.5110250790913899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,8192,0.03405333360036214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,10240,0.04756480058034261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,12288,0.05274986823399862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,16384,0.061457065741221104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,5120,0.02546986738840739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,6144,0.02848106622695923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,4096,0.018773333231608073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,7168,0.03173440098762512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,3584,0.017107200622558594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,3072,0.015742933750152587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,1536,0.009164800246556599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,2560,0.013404800494511922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,2048,0.01115733285744985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,768,0.006098133325576782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,1024,0.007099733253320058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,64,0.0034314667185147605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,256,0.004242133100827535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,512,0.005014400184154511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,128,0.003667200108369192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,32,0.003475199888149897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,65536,0.34754985173543296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,8192,0.026791467269261675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,10240,0.03354560136795044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,12288,0.0385696013768514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,16384,0.058589867750803624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,6144,0.021630932887395225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,4096,0.010381866494814556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,5120,0.017112533251444496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,7168,0.024013866980870567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,3584,0.010039466619491576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,1536,0.005678933362166087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,2560,0.007707733412583668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,2048,0.006426666676998138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,3072,0.008764800429344178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,1024,0.004925866425037384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,512,0.003922133396069208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,128,0.003221333275238673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,768,0.004318933188915253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,256,0.0034986667335033415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,64,0.0030986666679382324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,32,0.003311999887228012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,65536,0.2146399974822998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,8192,0.02950826684633891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,10240,0.03542186816533406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,12288,0.04132266839345296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,7168,0.0268885334332784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,16384,0.051489067077636716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,6144,0.021256534258524577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,5120,0.015608533223470052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,4096,0.008685866991678875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,3584,0.008571733037630718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,3072,0.008707200487454731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,2560,0.007496533294518788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,1024,0.0046858668327331545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,2048,0.006342400113741558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,1536,0.005814399818579356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,256,0.003435733417669932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,768,0.004148266712824504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,512,0.003756800045569738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,128,0.0031306666632493338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,64,0.003018666555484136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,32,0.0031328000128269195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,65536,0.1937407970428467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,8192,0.02823466658592224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,10240,0.033497599760691325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,12288,0.03994880119959514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,7168,0.02547520001729329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,16384,0.04993919928868611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,6144,0.022848000129063926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,5120,0.013755733768145243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,4096,0.008553600311279297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,2560,0.006525866687297821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,3072,0.006822399795055389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,3584,0.007446399827798207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,2048,0.0054517333706219995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,1536,0.005100800096988678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,512,0.003532800078392029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,768,0.003902933249870936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,1024,0.004251733422279358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,128,0.002997333308060964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,256,0.0031776001056035364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,64,0.0028938665986061097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,32,0.002962133288383484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,65536,0.18519147237141925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,65536,0.18065919876098632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,7168,2.5238998413085936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,6144,2.1443262736002606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,8192,2.8170069376627604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,10240,3.5306485493977866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,3072,1.047107187906901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,5120,1.7295402526855468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,4096,1.3763263702392579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,3584,1.2052715301513672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,12288,4.317992655436198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,768,0.2939551989237467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,512,0.2252565383911133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,1024,0.3778090794881185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,2048,0.7144287745157878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,2560,0.8751242955525717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,128,0.11767679850260418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,64,0.1159775972366333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,32,0.1193002700805664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,1536,0.5378559748331706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,256,0.14086079597473145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,16384,6.051584879557291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,7168,0.591316286722819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,8192,0.6707466761271159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,10240,0.8513130823771158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,12288,1.0187274932861328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,6144,0.5278474807739257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,4096,0.34239892959594725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,5120,0.42683839797973633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,3584,0.3058357238769531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,16384,1.387814458211263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,1024,0.09715200265248616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,3072,0.263428258895874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,2560,0.22666239738464355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,1536,0.1369717280069987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,2048,0.17772053082784017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,512,0.057265067100524904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,256,0.03760853211085002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,768,0.07837333679199218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,128,0.03747306664784749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,64,0.030529065926869707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,32,0.030750934282938642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,7168,0.44140160878499346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,8192,0.5056341489156086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,10240,0.6459274927775065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,12288,0.7545152028401693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,4096,0.2655584017435709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,5120,0.320743465423584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,16384,1.0044746398925781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,6144,0.38014399210611977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,3584,0.2298485279083252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,3072,0.20238720575968422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,1024,0.07368213335673014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,1536,0.10375786622365316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,2048,0.13606613477071125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,2560,0.16645545959472657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,768,0.059935998916625974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,512,0.04381973346074422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,128,0.02577706575393677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,256,0.029240532716115313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,64,0.024526933828989662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,32,0.02478826642036438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,8192,0.43842452367146806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,10240,0.5313791910807292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,12288,0.6319679896036784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,16384,0.8502890904744467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,7168,0.3854624112447103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,6144,0.3233194669087728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,5120,0.2697087923685709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,4096,0.22037119865417482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,3584,0.1987328052520752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,3072,0.16780266761779786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,2560,0.14104639689127604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,1536,0.09255359967549642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,2048,0.11688853104909261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,512,0.0392522652943929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,1024,0.0636234680811564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,768,0.050780800978342686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,128,0.022245333592096964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,256,0.02785173257191976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,64,0.019832533597946168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,32,0.020116267601648967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,65536,5.6708831787109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,10240,0.41841920216878253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,12288,0.5033077239990235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,65536,4.356813049316406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,16384,0.6712352116902669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,8192,0.34073705673217775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,6144,0.256114133199056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,7168,0.29536641438802086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,4096,0.1750666618347168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,5120,0.21964693069458008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,3584,0.15286720593770345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,1536,0.07198826471964517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,1024,0.05034559965133667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,2048,0.0928874651590983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,2560,0.1120789368947347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,3072,0.13339519500732422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,768,0.041169067223866776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,32,0.01551040013631185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,512,0.03151999910672505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,256,0.021915733814239502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,128,0.01691626707712809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,64,0.014945066968599954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,65536,3.543293762207031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,8192,0.2949301401774088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,10240,0.3644981384277344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,12288,0.440884272257487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,16384,0.5805418650309245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,6144,0.22408533096313477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,7168,0.26450133323669434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,4096,0.15238399505615235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,5120,0.18655786514282227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,2048,0.0779754638671875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,1536,0.05922026634216308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,2560,0.0962719996770223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,3584,0.1322655995686849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,3072,0.11462612946828206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,1024,0.04210986693700154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,512,0.02480213244756063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,768,0.033046400547027587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,128,0.014667733510335287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,256,0.018333866198857626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,64,0.013612799843152366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,32,0.014204800128936768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,65536,2.8210474650065107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,8192,0.2562741279602051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,10240,0.3181920051574707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,12288,0.376584529876709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,7168,0.23223360379536948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,16384,0.5019466718037923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,6144,0.19399466514587402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,5120,0.1633834679921468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,4096,0.1309557358423869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,3584,0.11947733561197918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,3072,0.10117866992950439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,2560,0.08590826988220215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,2048,0.07051093578338623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,1536,0.05402666727701823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,1024,0.0401962677637736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,768,0.032177066802978514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,64,0.012173866232236225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,256,0.017833600441614784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,512,0.024918399254480996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,128,0.014069333672523499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,32,0.012588799993197123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,65536,2.439891306559245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,8192,0.21281065940856933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,10240,0.2633685270945231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,12288,0.31765546798706057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,16384,0.42072534561157227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,7168,0.19079999923706054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,6144,0.16138346989949542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,4096,0.11034026940663655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,5120,0.13469759623209637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,3072,0.08452693621317545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,3584,0.09737919966379802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,2560,0.07121919790903727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,2048,0.05857919851938883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,1536,0.045076266924540205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,1024,0.03301440080006917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,768,0.02604373296101888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,256,0.014350933829943338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,64,0.009297066926956176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,512,0.019885865847269694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,128,0.010571733117103577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,32,0.00944533348083496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,65536,1.9989002227783204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,8192,0.16929066975911458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,10240,0.2084554672241211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,12288,0.25001279513041175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,16384,0.3344031969706217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,6144,0.12840747038523356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,4096,0.08721386591593425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,7168,0.14854507446289061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,5120,0.10696319739023845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,3584,0.07775893211364746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,1024,0.025757867097854614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,3072,0.06726933320363362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,2048,0.04591466585795085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,1536,0.03576106627782186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,2560,0.05642133156458536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,768,0.02092906634012858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,512,0.016364799936612447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,256,0.011363200346628825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,32,0.008169599870840708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,64,0.007653333246707916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,128,0.008787199854850769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,65536,1.7554250081380207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,8192,0.1598218599955241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,10240,0.1998645305633545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,12288,0.24105280240376792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,16384,0.31415786743164065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,7168,0.14653973579406737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,4096,0.08382933139801026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,6144,0.12126932938893635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,5120,0.102565336227417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,2560,0.05451840162277222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,3072,0.06517546574274699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,3584,0.07283093134562174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,1536,0.034619732697804766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,2048,0.04449066718419393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,1024,0.02533866763114929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,768,0.020795732736587524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,256,0.011313066879908244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,512,0.016174933314323424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,128,0.008502399921417237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,64,0.007313066720962524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,32,0.00784746656815211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,65536,1.3232308705647786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,8192,0.1346901257832845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,10240,0.16453973452250165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,12288,0.1936192035675049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,7168,0.11622933546702068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,16384,0.2561386744181315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,5120,0.08450667063395181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,6144,0.10205439726511638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,4096,0.06734933058420817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,3584,0.06023146708806356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,3072,0.05140693187713623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,2560,0.04414933522542318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,2048,0.03679573138554891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,1536,0.02759573260943095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,1024,0.02071253259976705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,256,0.008988799651463826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,512,0.012857600053151449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,768,0.016646400094032288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,128,0.007228800157705943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,32,0.00664106657107671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,64,0.006481066842873891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,65536,1.2775508880615234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,8192,0.10630079905192058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,10240,0.1342463970184326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,12288,0.15565333366394044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,7168,0.09447253545125325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,16384,0.205238405863444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,5120,0.06937493483225504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,6144,0.08083519935607911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,3584,0.04906133413314819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,4096,0.05465813477834066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,2560,0.036652799447377524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,3072,0.0422111988067627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,2048,0.030092799663543703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,768,0.014614400267601014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,1536,0.023681066433588662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,1024,0.01742186745007833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,128,0.00689279983441035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,512,0.01107413371404012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,256,0.008526933193206788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,64,0.006141866743564606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,32,0.006378666559855144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,65536,1.0225866953531901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,8192,0.083678936958313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,10240,0.1046453317006429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,7168,0.07491947015126546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,12288,0.12501973311106365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,16384,0.16295893987019855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,5120,0.05450559854507446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,6144,0.0639573335647583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,4096,0.044308265050252274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,3584,0.039094400405883786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,3072,0.033589335282643636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,2560,0.028968532880147297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,1536,0.019206400712331137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,2048,0.02376746733983358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,1024,0.014723199605941772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,768,0.011849600076675414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,512,0.009668266773223877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,256,0.00718506673971812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,128,0.0060245335102081295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,64,0.005435733497142792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,32,0.005539200206597646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,65536,0.8238847732543946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,7168,0.05873493353525797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,8192,0.0651914676030477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,10240,0.08175466855367025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,12288,0.0967146635055542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,6144,0.05040640036265055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,16384,0.1279221296310425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,5120,0.041980799039204916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,4096,0.03470933437347412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,3072,0.02659306724866231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,3584,0.030009599526723225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,2048,0.018745599190394084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,2560,0.02263573408126831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,1536,0.015508266290028891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,1024,0.011374933520952861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,768,0.009451733032862345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,64,0.004654933512210846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,256,0.006095999975999197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,512,0.00766186664501826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,128,0.005261866748332978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,32,0.004859733581542969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,65536,0.6648341496785481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,7168,0.0407914678255717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,8192,0.04495360056559245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,10240,0.05611093441645304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,12288,0.06596159934997559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,6144,0.03543146848678589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,16384,0.08713706334431967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,5120,0.02911466757456462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,4096,0.02378773291905721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,3584,0.020588799317677816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,3072,0.018380800882975258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,2560,0.01597760021686554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,768,0.0073173334201176955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,2048,0.01332373321056366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,1536,0.011220266421635944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,1024,0.008363733688990276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,512,0.006197333335876465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,256,0.005099733173847198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,64,0.004063999901215235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,128,0.0044064000248909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,32,0.0041354666153589886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,65536,0.5063434600830078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,8192,0.035164801279703777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,7168,0.03159573276837667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,10240,0.047416532039642335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,12288,0.05849173466364542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,16384,0.07031573454538981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,3072,0.01644159952799479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,6144,0.032509867350260416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,3584,0.018517333269119262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,5120,0.025443200270334882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,4096,0.020942932367324828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,2560,0.01388800044854482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,1024,0.007197866837183635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,768,0.006265600025653839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,1536,0.009388800462086995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,2048,0.011770666639010111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,512,0.005438933273156484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,65536,0.34228798548380535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,256,0.0045952002207438145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,64,0.0037087999284267426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,128,0.0040224000811576845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,32,0.0037952000896135964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,7168,0.029283199707667035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,8192,0.03153706590334575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,12288,0.04862506786982219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,10240,0.0452021320660909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,16384,0.05277119874954224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,6144,0.025896533330281572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,4096,0.017065600554148356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,3072,0.01421440045038859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,5120,0.02299413283665975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,3584,0.0157642662525177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,1536,0.008262399832407634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,768,0.005660800139109293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,2048,0.01016533374786377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,1024,0.006167466441790262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,2560,0.011914666493733723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,512,0.004741333425045013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,65536,0.26459412574768065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,128,0.003703466554482778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,256,0.00413226659099261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,64,0.0033642667035261786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,32,0.0034506666163603462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,7168,0.01902186671892802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,8192,0.020916267236073812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,10240,0.0304202675819397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,12288,0.031078400214513142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,16384,0.03742826779683431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,5120,0.013527466853459676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,6144,0.01718826691309611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,3584,0.009428266684214275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,4096,0.01142080028851827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,3072,0.009261866410573322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,1536,0.005681066711743673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,65536,0.1866165320078532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,1024,0.004949333270390829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,2048,0.006387199958165486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,768,0.004373333354791006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,2560,0.007811200122038524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,256,0.0035317334036032355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,512,0.003949866692225138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,32,0.0032405334214369455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,128,0.0032778667906920114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,64,0.0030432000756263735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,7168,0.01291306714216868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,8192,0.014934399724006652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,10240,0.021862399578094483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,12288,0.025546665986378985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,16384,0.03183253407478333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,6144,0.015063466628392539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,5120,0.014447999993960061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,3584,0.009922132889429728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,3072,0.00928106705347697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,4096,0.010652800401051838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,65536,0.1144757350285848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,768,0.004188799858093261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,1024,0.004625066618124644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,1536,0.005560533205668131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,512,0.0037589333951473236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,2048,0.006443733473618825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,2560,0.007057066758473713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,256,0.0033610666791598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,32,0.003067733347415924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,64,0.0029696000119050344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,128,0.0031925333042939507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,8192,0.011592533191045125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,7168,0.010759466886520385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,10240,0.019828265905380248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,12288,0.023893332481384276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,16384,0.02927466630935669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,5120,0.009085866808891296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,6144,0.015287466843922935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,4096,0.008263466755549113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,3584,0.007429333527882893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,3072,0.0067893331249554946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,1536,0.004991999765237173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,2048,0.005621333420276642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,65536,0.10446720123291016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,2560,0.006435200075308482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,1024,0.004231466849644979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,512,0.0035743998984495797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,768,0.0038794666528701783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,256,0.003236266722281774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,64,0.0029450667401154833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,128,0.0029696000119050344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,32,0.0028405333558718365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,10240,0.015961600343386333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,7168,0.009884799520174663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,8192,0.010318932930628459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,12288,0.022818134228388468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,16384,0.02805333336194356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,6144,0.008881066242853801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,5120,0.008215466638406117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,4096,0.007483733197053273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,3584,0.007181866466999054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,2560,0.006213333209355672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,3072,0.006851199766000111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,1024,0.004232533276081085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,2048,0.005430399874846141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,65536,0.09987839857737223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,1536,0.004839466512203216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,768,0.0038794666528701783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,512,0.003542399903138479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,256,0.0032543999453385672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,128,0.003035733352104823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,64,0.00297173336148262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,32,0.003052799900372823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,65536,0.09600213368733725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,6144,1.0611093521118165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,7168,1.2471808115641276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,5120,0.8930325190226236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,8192,1.4407061258951823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,10240,1.811362075805664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,3072,0.5450954437255859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,2560,0.4573845227559407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,4096,0.7093215942382812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,3584,0.6284287770589192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,12288,2.1429824829101562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,2048,0.37945388158162435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,512,0.115611735979716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,768,0.15903786023457844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,1536,0.28527679443359377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,1024,0.19724480311075848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,128,0.06363626718521118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,256,0.08165120283762614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,32,0.057722667853037514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,64,0.05842560132344564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,16384,2.972465006510417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,7168,0.3010464032491048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,8192,0.34003947575887045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,10240,0.4220992088317871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,12288,0.5111370722452799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,6144,0.2693098704020182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,5120,0.21809600194295248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,4096,0.17488640149434406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,3584,0.1540554682413737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,16384,0.6737183888753255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,3072,0.13533867200215657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,2560,0.11670506795247396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,512,0.03222080071767171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,768,0.04264640013376872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,1024,0.0526090661684672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,1536,0.07173759937286377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,2048,0.09363199869791666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,256,0.022983467578887938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,128,0.017863466342290243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,64,0.014944000045458474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,32,0.015513599912325541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,7168,0.22376426060994467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,8192,0.2565354665120443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,10240,0.32005653381347654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,12288,0.3900522549947103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,16384,0.5056597391764324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,4096,0.13326400121053059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,5120,0.16344960530598956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,3584,0.11999893188476562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,6144,0.19278507232666015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,1024,0.03932693401972453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,1536,0.05643733342488607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,2048,0.07122986316680908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,2560,0.0870837370554606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,512,0.025497599442799883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,3072,0.10208853085835774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,768,0.032022400697072344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,64,0.012739200393358865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,128,0.014206932981808982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,256,0.01795733372370402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,32,0.0126720001300176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,8192,0.21676586469014486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,10240,0.2680458704630534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,12288,0.3221909205118815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,16384,0.43622506459554033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,6144,0.16383786201477052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,7168,0.19074773788452148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,5120,0.13825066884358722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,4096,0.11411733627319336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,2560,0.07278933525085449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,3072,0.08580693403879801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,3584,0.09876800378163655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,2048,0.0605183998743693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,1536,0.04691093365351359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,512,0.020175999402999877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,768,0.026768000920613606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,1024,0.03365653355916341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,128,0.011102933684984844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,256,0.014285866419474283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,65536,2.8832374572753907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,64,0.00914026697476705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,32,0.009662933150927226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,8192,0.17166080474853515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,10240,0.21194240252176919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,12288,0.2589130719502767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,65536,2.025165812174479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,16384,0.3373471895853678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,5120,0.11032640139261882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,6144,0.12937386830647785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,7168,0.15109866460164387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,3584,0.07844693660736084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,4096,0.08939092953999837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,3072,0.06817493438720704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,2048,0.047048532962799074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,2560,0.057353599866231286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,1536,0.0370688001314799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,768,0.021064533789952596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,1024,0.026702932516733807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,512,0.01636799971262614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,64,0.007716266810894013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,256,0.011819733182589214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,128,0.008877866466840108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,32,0.008020266890525818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,65536,1.739079411824544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,8192,0.16182400385538737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,10240,0.20318826039632162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,12288,0.24147520065307618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,16384,0.32275733947753904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,7168,0.14479467074076335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,5120,0.10455146630605061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,6144,0.12252906958262126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,4096,0.08411093552907309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,3584,0.07482773462931315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,3072,0.0640234669049581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,1024,0.025388799111048382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,1536,0.03520426750183105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,2048,0.04503999948501587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,2560,0.05491199890772501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,512,0.01607039968172709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,768,0.021109332640965782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,32,0.007921066880226136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,128,0.008529067039489746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,256,0.011777066191037496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,64,0.007227733234564463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,65536,1.3963520050048828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,8192,0.13316906293233235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,10240,0.16452372868855794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,12288,0.19567039807637532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,7168,0.12083733081817627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,16384,0.26318186124165854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,6144,0.10105600357055664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,5120,0.08457600275675456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,4096,0.06825386683146159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,3072,0.05298453172047933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,2560,0.04495146671930949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,3584,0.062057598431905114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,1024,0.020343466599782308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,2048,0.036755200227101645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,1536,0.028135466575622558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,256,0.009271466732025146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,768,0.016639999548594155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,512,0.013266133268674216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,128,0.0071936001380284624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,32,0.006619733572006225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,64,0.006404266754786174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,65536,1.2786443074544271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,8192,0.10687572956085205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,10240,0.13394986788431804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,12288,0.15888427098592123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,16384,0.21545707384745277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,7168,0.09655573368072509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,5120,0.0694165309270223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,6144,0.08124480247497559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,4096,0.055949866771698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,3584,0.05068373282750448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,3072,0.04325973192850749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,2560,0.03689173460006714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,1536,0.023638399442036946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,2048,0.03011946678161621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,512,0.011921067039171855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,1024,0.017858133713404337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,768,0.014801067113876343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,256,0.008344533046086629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,64,0.006393600006898243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,128,0.006995200117429097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,32,0.006557866434256236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,65536,1.0294645309448243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,8192,0.08749120235443116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,10240,0.10545813242594401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,12288,0.12708373069763185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,7168,0.07595093250274658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,16384,0.16675413449605306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,4096,0.04498240152994792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,6144,0.06698453426361084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,3584,0.03941973447799683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,5120,0.05459200143814087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,2560,0.029731200138727827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,3072,0.03456533352533976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,2048,0.02482773264249166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,1536,0.019215999046961467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,768,0.012311466534932454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,1024,0.014578133821487427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,512,0.009641599655151368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,256,0.007188266515731812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,64,0.005412266651789347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,128,0.006072533130645752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,32,0.0056085333228111265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,65536,0.8342453638712565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,7168,0.07247146765391031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,12288,0.11994667053222656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,10240,0.09896213213602702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,8192,0.08006186485290527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,16384,0.16132906277974446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,4096,0.039496533075968426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,5120,0.05054933230082194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,3584,0.03479359944661458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,6144,0.059425067901611325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,3072,0.03127146760622661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,2048,0.021259733041127524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,1024,0.01239359974861145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,1536,0.017538134256998697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,2560,0.025573333104451496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,512,0.007743999858697255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,768,0.010376532872517902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,256,0.006171733140945435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,65536,0.6594367980957031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,128,0.005356800059477488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,64,0.004960000018278758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,32,0.0052714665730794275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,7168,0.05935360193252563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,8192,0.06601599852244058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,10240,0.08222506841023763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,12288,0.09949226379394531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,6144,0.05118293364842733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,16384,0.1301205317179362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,5120,0.042583465576171875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,3072,0.02718399961789449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,3584,0.031111466884613036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,4096,0.03476159969965617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,2560,0.023112533489863078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,1536,0.01529813309510549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,2048,0.018925867478052773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,512,0.0077909335494041445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,1024,0.01167680025100708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,768,0.009994666775067646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,256,0.00609493354956309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,65536,0.6186506907145183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,64,0.0046858668327331545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,128,0.005138133466243744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,32,0.004890666902065277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,7168,0.05076906681060791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,16384,0.111789870262146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,8192,0.05702293316523234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,10240,0.07127253214518228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,12288,0.08475306828816732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,6144,0.044093867142995194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,5120,0.037620266278584794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,3072,0.02363199989000956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,3584,0.027378133932749432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,4096,0.030352000395456952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,2048,0.01723840037981669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,2560,0.020470400651295982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,1024,0.011358933647473653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,1536,0.014318933089574179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,768,0.009337600072224934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,65536,0.5262314796447753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,512,0.007762133578459422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,128,0.005075199902057648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,64,0.004582400123278299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,256,0.005907199780146281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,32,0.004766933123270671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,7168,0.03917653163274129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,8192,0.04485973517100016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,16384,0.08719360033671061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,10240,0.05551040172576904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,12288,0.06647573312123617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,4096,0.02369920015335083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,6144,0.034142935276031496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,5120,0.02934933304786682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,3072,0.018284799655278523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,3584,0.020951465765635172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,2560,0.01585493286450704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,1024,0.008529067039489746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,768,0.007259733478228251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,512,0.006153599917888641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,1536,0.01114026705423991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,2048,0.013474133610725404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,65536,0.4309269269307454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,64,0.004011733333269755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,256,0.005058133105436961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,128,0.004410666724046072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,32,0.004260266820589701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,12288,0.05144000053405762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,8192,0.03476053476333618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,10240,0.04319146474202474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,7168,0.03323413332303365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,16384,0.06688533624013265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,6144,0.026899200677871705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,4096,0.01829866568247477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,5120,0.0228767991065979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,3584,0.017785600821177163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,3072,0.01469013293584188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,2560,0.012831999858220419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,65536,0.3491711934407552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,2048,0.010915199915568035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,1536,0.009083732962608337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,1024,0.007122133175532024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,512,0.005356800059477488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,768,0.00624533345301946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,256,0.004589866598447164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,128,0.004040533304214477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,64,0.0036415999134381616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,32,0.0038560000558694207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,12288,0.05017600059509277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,7168,0.029640533526738483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,8192,0.03158933321634928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,10240,0.04195839961369832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,16384,0.05079466501871745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,6144,0.02730453411738078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,3584,0.015821866194407144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,4096,0.016707199811935424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,5120,0.022027732928593953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,2560,0.011819733182589214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,65536,0.2627328077952067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,3072,0.013513599832852682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,768,0.005529599885145823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,2048,0.00963200032711029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,1536,0.00767146646976471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,1024,0.006316799918810527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,512,0.004739200075467428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,128,0.0036757332583268487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,256,0.004066133250792822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,64,0.003403733422358831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,32,0.0034346667428811393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,12288,0.03950933218002319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,10240,0.035699200630187986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,8192,0.023924267292022704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,7168,0.024489599466323852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,16384,0.04084159930547078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,65536,0.17886719703674317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,6144,0.021130667130152384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,4096,0.013863466183344521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,3584,0.012991999586423239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,5120,0.01732053359349569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,3072,0.011239467064539592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,2048,0.008091733356316884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,2560,0.009925333658854167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,1536,0.007203199962774913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,1024,0.005336533486843109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,768,0.0047765334447224935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,512,0.004195199906826019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,256,0.003688533355792364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,128,0.0033514666060606635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,64,0.0031413334111372627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,32,0.0032831999162832894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,10240,0.029790933926900225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,8192,0.020773333311080933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,7168,0.019569067160288493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,12288,0.03091946641604106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,16384,0.032671999931335446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,65536,0.13854187329610188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,3584,0.010318932930628459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,5120,0.013685333728790283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,6144,0.017670400937398276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,4096,0.011724799871444702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,3072,0.009729066491127014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,2560,0.008102400104204814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,2048,0.006965333223342895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,1024,0.004814933240413666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,1536,0.005713066458702088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,768,0.0043712000052134195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,512,0.0038794666528701783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,64,0.0031839999059836066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,128,0.003356799980004629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,256,0.0035264000296592714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,32,0.0031786667803923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,7168,0.013081600268681845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,10240,0.018613332509994508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,8192,0.014310399691263834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,12288,0.019777067502339683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,65536,0.09716479778289795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,16384,0.02285760045051575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,6144,0.01358080009619395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,3584,0.008307200173536937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,5120,0.012414933244387309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,4096,0.009746133287747701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,2560,0.007121066749095917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,3072,0.007799466451009114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,1536,0.005656533439954122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,2048,0.0064416001240412395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,1024,0.004825599988301595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,768,0.004285866518815359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,128,0.0031829332311948144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,512,0.0037813333173592886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,256,0.0033973333736260734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,64,0.00290133332212766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,32,0.003110400090614955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,7168,0.012350933750470479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,8192,0.008746666709582011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,65536,0.06663680076599121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,10240,0.010846933722496033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,12288,0.011450666189193725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,16384,0.015101866920789084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,5120,0.008975999553998311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,6144,0.008760533730189006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,4096,0.0081535999973615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,2560,0.006355200211207073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,3072,0.00795413355032603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,3584,0.008374399940172831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,768,0.003907199949026108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,2048,0.005612800021966299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,1536,0.005005866785844167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,1024,0.004275199770927429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,512,0.0035264000296592714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,256,0.0032543999453385672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,128,0.003047466774781545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,64,0.002856533229351044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,32,0.002997333308060964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,65536,0.05713280042012533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,8192,0.007628799974918365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,7168,0.00761599987745285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,10240,0.00844053328037262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,12288,0.009990400075912476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,16384,0.01179093321164449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,4096,0.007540266712506611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,3584,0.007155199845631917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,6144,0.008850133419036866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,5120,0.008169599870840708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,2560,0.006200533111890157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,3072,0.00681279997030894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,1536,0.004839466512203216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,2048,0.0055071999629338585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,1024,0.004170666635036469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,768,0.003818666686614355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,512,0.003458133339881897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,256,0.003193599979082743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,128,0.003031466652949651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,64,0.003089066594839096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,32,0.0030005333324273427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,7168,0.007197866837183635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,8192,0.007506133119265239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,10240,0.008520533641179402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,65536,0.056269868214925134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,12288,0.009381332993507385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,16384,0.01076693336168925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,5120,0.00702400008837382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,4096,0.00660159985224406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,3584,0.006975999971230824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,6144,0.006597333153088887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,3072,0.006595199803511302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,2560,0.006055466830730438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,2048,0.005454933146635691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,1536,0.004756266872088114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,1024,0.004106666644414266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,768,0.0038474666575590765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,512,0.0034400001168251038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,256,0.003017599880695343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,128,0.00327360009153684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,64,0.0028319999575614927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,32,0.0027776000400384264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,65536,0.053931732972462974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,5120,0.6565439860026042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,6144,0.7871018727620442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,7168,0.9232490539550782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,8192,1.0617866516113281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,3072,0.40470399856567385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,10240,1.307044219970703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,3584,0.46511999766031903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,4096,0.5309322675069172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,2560,0.34764267603556315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,768,0.11917760372161865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,512,0.08691946665445963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,1024,0.15052053133646648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,1536,0.21255253156026205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,128,0.05040426651636759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,12288,1.5454911549886068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,2048,0.27685333887736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,64,0.04527573188145955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,256,0.05781973203023275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,32,0.045415465037027994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,7168,0.2241151968638102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,8192,0.25832746823628744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,10240,0.3194911956787109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,16384,2.1101898193359374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,12288,0.3840298652648926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,5120,0.16453866958618163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,3072,0.10308372974395752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,16384,0.5042943954467773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,6144,0.19348160425821942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,4096,0.1326848030090332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,3584,0.12009706497192382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,2048,0.07170133590698242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,768,0.032730666796366374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,512,0.02582506736119588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,2560,0.0869493325551351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,1024,0.03998719851175944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,1536,0.0565280000368754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,256,0.017910399039586387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,128,0.014016000429789224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,64,0.013172266880671182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,32,0.013033599654833476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,7168,0.16813546816507977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,8192,0.1910912036895752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,10240,0.2359018643697103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,12288,0.2846144040425619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,3584,0.08743253548940023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,16384,0.37257919311523435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,6144,0.144814936319987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,5120,0.12192959785461426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,4096,0.0980522632598877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,1024,0.02909226616223653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,3072,0.07593706448872885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,1536,0.041137067476908366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,2048,0.0525877316792806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,2560,0.06423999865849814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,768,0.023477333784103393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,512,0.018017067511876424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,256,0.013084800044695536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,128,0.010163199901580811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,32,0.00879146655400594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,64,0.008483200271924337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,8192,0.1632479985555013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,10240,0.2042912006378174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,12288,0.24453120231628417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,16384,0.3245941480000814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,7168,0.14351785977681478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,6144,0.12494293053944905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,5120,0.10516160329182941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,4096,0.08504106998443603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,3072,0.06533973217010498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,2560,0.0556714653968811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,3584,0.07465066909790039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,1024,0.025796266396840413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,2048,0.045672531922658285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,65536,2.1158101399739584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,1536,0.03576320012410482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,768,0.020866133769353232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,128,0.010833066701889039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,512,0.016218666235605875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,64,0.009854933619499207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,256,0.011747200290362041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,32,0.010545066992441813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,8192,0.1310805320739746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,65536,1.4739028930664062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,10240,0.1643114725748698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,12288,0.19504213333129883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,16384,0.2601045290629069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,5120,0.0843989372253418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,7168,0.11532800197601319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,4096,0.06809813181559245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,6144,0.10072426795959473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,3584,0.060813868045806886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,1536,0.02845653295516968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,1024,0.02034133275349935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,2048,0.03633706569671631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,2560,0.04430400133132935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,3072,0.05268266598383585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,768,0.016697599490483602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,512,0.01311360001564026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,256,0.009341866771380106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,64,0.006479999919732411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,128,0.007457066575686138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,65536,1.3042357126871744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,32,0.006726400057474773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,7168,0.10124693711598713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,8192,0.11843199729919433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,10240,0.14239360491434733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,12288,0.17137707074483235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,16384,0.22629547119140625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,5120,0.0748309294382731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,6144,0.08760746320088705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,4096,0.05964906613032023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,3584,0.0533461332321167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,2560,0.03964693148930867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,3072,0.045932801564534505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,768,0.014921599626541137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,2048,0.03179093400637309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,1536,0.02477226654688517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,1024,0.01816533406575521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,256,0.008930133779843648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,512,0.011290666460990906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,128,0.007804800073305767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,64,0.007276799778143566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,32,0.0077344000339508055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,65536,1.0584704081217446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,8192,0.09575573603312174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,10240,0.1172106663386027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,7168,0.08756693204243979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,12288,0.13980693817138673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,16384,0.18623894055684406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,4096,0.05128106673558554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,6144,0.07292799949645996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,3584,0.043901864687601724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,5120,0.06074986855189005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,3072,0.03857066631317139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,2560,0.03350720008214315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,2048,0.027056000630060834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,1536,0.021373866001764934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,1024,0.01601920028527578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,512,0.010798933108647664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,768,0.013510400056838989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,32,0.006131199995676676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,256,0.007866666714350382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,128,0.0065749332308769224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,64,0.005769599974155426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,65536,0.8984874725341797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,8192,0.08287893136342367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,10240,0.10242239634195964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,12288,0.12333973248799641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,7168,0.0750378688176473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,16384,0.16700480779012045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,3072,0.03456213474273682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,6144,0.06318613290786743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,5120,0.053301334381103516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,3584,0.038983468214670816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,4096,0.04375040133794149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,2560,0.029473066329956055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,1024,0.014754133423169455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,512,0.009738666812578838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,2048,0.023820799589157105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,1536,0.019182932376861573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,768,0.012167466680208842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,128,0.006085333228111267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,256,0.007428266604741414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,65536,0.7441002527872722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,64,0.00535999983549118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,32,0.00556160012880961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,7168,0.05821119944254557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,10240,0.08430933157602946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,8192,0.06610133250554404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,12288,0.09873173236846924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,16384,0.13271466890970868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,5120,0.043323731422424315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,3584,0.031251200040181476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,4096,0.034652801354726155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,6144,0.05034026702245077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,3072,0.027586134274800618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,2560,0.023149865865707397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,2048,0.019285333156585694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,768,0.009861333171526591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,1024,0.011764267086982727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,1536,0.015294933319091797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,512,0.00791786660750707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,256,0.006209066510200501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,65536,0.6413461049397786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,128,0.005291733145713806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,64,0.004642133414745331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,32,0.004917333523432413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,10240,0.07435413201649985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,7168,0.05141760110855102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,8192,0.05837653477986654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,12288,0.08754879633585612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,16384,0.113810133934021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,6144,0.04606826702753703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,4096,0.03109440008799235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,3584,0.02782613237698873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,5120,0.03788160085678101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,3072,0.024567466974258424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,2560,0.021317332983016968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,2048,0.017684266964594523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,1536,0.014419200023015341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,512,0.007811200122038524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,1024,0.01132480005423228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,768,0.010270933310190838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,65536,0.5113408088684082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,64,0.005050666630268097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,256,0.006580266853173573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,128,0.005539200206597646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,32,0.005293866495291392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,8192,0.05045440196990967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,7168,0.044468267758687334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,12288,0.07651200294494628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,10240,0.06205653349558512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,16384,0.09885120391845703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,4096,0.027525333563486735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,5120,0.03408746719360352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,3584,0.024234666426976522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,3072,0.02138026754061381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,6144,0.038701868057250975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,2560,0.018295466899871826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,65536,0.4480725288391113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,1536,0.01237440009911855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,2048,0.015661866466204325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,1024,0.00981119970480601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,768,0.008113066852092742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,512,0.006595199803511302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,256,0.005491200089454651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,128,0.004682666560014089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,32,0.004436266422271728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,64,0.004229333500067393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,12288,0.06508586804072061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,7168,0.03863999843597412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,8192,0.04408106803894043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,16384,0.08733332951863607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,10240,0.05389866828918457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,6144,0.03415466547012329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,4096,0.02346986730893453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,3584,0.02145386735598246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,5120,0.028429865837097168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,3072,0.01821333368619283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,65536,0.38548161188761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,1536,0.011198932925860088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,2560,0.01602666676044464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,2048,0.013434666395187377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,768,0.007379200061162312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,1024,0.008839466174443563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,512,0.006284800171852112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,256,0.005018666883309682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,128,0.004452266792456309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,64,0.004193066557248434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,32,0.004332800209522247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,7168,0.04224426746368408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,8192,0.045716265837351486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,10240,0.060838401317596436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,12288,0.07336533069610596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,16384,0.09375360012054443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,4096,0.02038080096244812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,3584,0.018655999501546224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,5120,0.031096533934275312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,6144,0.03700266679128011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,65536,0.33496532440185545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,3072,0.016862932840983072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,2560,0.015027200182278952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,2048,0.012258133292198181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,1536,0.010233599940935771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,1024,0.0073183998465538025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,512,0.0053962667783101406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,768,0.006246399879455566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,256,0.004470400015513102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,128,0.0039818666875362395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,64,0.003669333209594091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,32,0.0038176000118255613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,10240,0.03637973467508952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,12288,0.042430933316548666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,7168,0.025562665859858197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,8192,0.028943999608357744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,16384,0.05824533303578695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,6144,0.02296853264172872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,5120,0.018946133057276406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,4096,0.015448533495267234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,3584,0.01404800017674764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,65536,0.26078933080037436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,3072,0.012532266974449157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,2048,0.012400000294049581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,2560,0.011202133695284526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,1024,0.007474133372306823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,1536,0.010041600465774536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,768,0.006381866832574208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,256,0.004173866907755534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,128,0.003664000084002813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,512,0.0051818668842315676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,64,0.003568000098069509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,32,0.0036490666369597114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,12288,0.049854934215545654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,10240,0.04296640157699585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,16384,0.041630931695302326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,8192,0.03530453443527222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,7168,0.03130346735318502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,65536,0.19979626337687176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,6144,0.025732266902923583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,4096,0.013900799552599588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,5120,0.018248534202575682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,3072,0.011296000083287556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,3584,0.0127893328666687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,1536,0.0070783997575442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,2560,0.00937066674232483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,2048,0.00844266712665558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,1024,0.0055754666527112326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,768,0.004885333279768625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,512,0.004353066782156626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,256,0.0037770666182041167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,64,0.0033290666838486993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,128,0.0034346667428811393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,32,0.003340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,10240,0.023421865701675416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,7168,0.016833066940307617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,12288,0.027850667635599773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,8192,0.0189301331837972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,16384,0.034732798735300704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,65536,0.13815679550170898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,4096,0.010904533664385478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,6144,0.0149536003669103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,5120,0.013079466422398886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,3584,0.01018346647421519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,3072,0.0090421328941981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,2560,0.011345066626866658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,1536,0.007863466441631318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,2048,0.009583999713261921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,1024,0.006085333228111267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,768,0.005309866865475973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,512,0.004411733150482178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,256,0.003773866593837738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,128,0.0033962666988372804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,64,0.0032981333633263906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,32,0.0031487998863061273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,8192,0.01624853312969208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,7168,0.014978133638699851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,10240,0.01980479955673218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,12288,0.021821866432825722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,65536,0.11130987008412678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,16384,0.027428267399470012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,6144,0.013247999548912048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,4096,0.00977066655953725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,5120,0.011741866668065388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,3584,0.009101866682370504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,2560,0.0076223999261856076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,3072,0.008150400221347808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,2048,0.006132266422112783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,1536,0.005752533177534739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,1024,0.004644266764322917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,768,0.004197333256403605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,512,0.003841066608826319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,256,0.0034058667719364167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,128,0.0032874666154384612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,64,0.0031008000175158186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,32,0.0031093334158261614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,65536,0.0791541337966919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,8192,0.012635733683904013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,10240,0.01441493332386017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,7168,0.011742933591206869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,12288,0.015884799758593242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,16384,0.01771946748097738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,5120,0.009987200299898785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,6144,0.011517866452534994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,4096,0.008574933807055155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,3584,0.007786666850248973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,3072,0.007044266661008198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,2560,0.006636799871921539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,1024,0.00444160004456838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,2048,0.005861333509286245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,1536,0.005208533505598704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,768,0.004060799876848856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,256,0.003328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,512,0.003676799933115641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,128,0.0031776001056035364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,64,0.0029834667841593427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,32,0.0029685333371162414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,65536,0.05624320109685262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,8192,0.00944533348083496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,12288,0.012029866377512615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,10240,0.010878933469454448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,7168,0.00906773308912913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,16384,0.011380267143249512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,5120,0.00839573343594869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,3584,0.00764160007238388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,6144,0.008516266942024231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,3072,0.006951466699441274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,4096,0.007709866762161255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,2560,0.006251733501752217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,1536,0.004859733581542969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,2048,0.00555626650651296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,768,0.003861333429813385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,1024,0.0041685332854588825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,512,0.003472000112136205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,256,0.0031487998863061273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,128,0.0030389333764712016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,64,0.002881066749493281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,32,0.002903466671705246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,65536,0.05528106689453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,12288,0.01034346620241801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,8192,0.008428800106048583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,10240,0.009425066908200582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,16384,0.012505599856376648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,7168,0.008107733229796093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,6144,0.007527466615041096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,4096,0.006669866542021434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,3584,0.007098666826883952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,5120,0.0071381335457166035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,2048,0.005491200089454651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,2560,0.00621973325808843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,3072,0.006828799843788147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,1024,0.004091733445723852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,1536,0.0048096001148223875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,768,0.0037813333173592886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,512,0.0034453332424163817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,256,0.003124266614516576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,128,0.0029290666182835894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,64,0.002915200094381968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,32,0.002942933390537898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,65536,0.04578453302383423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,7168,0.00681279997030894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,8192,0.007243733108043671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,12288,0.00843946635723114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,10240,0.007866666714350382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,16384,0.00960533320903778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,6144,0.006578133503595989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,4096,0.00661653329928716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,5120,0.006995200117429097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,2560,0.006097066899140676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,3584,0.006971733272075653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,3072,0.006592000027497609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,2048,0.005413333574930826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,1024,0.004084266722202301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,1536,0.004757333298524221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,768,0.003706666578849157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,512,0.003416533271471659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,256,0.003018666555484136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,128,0.002917333443959554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,64,0.0027989332874615988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,32,0.002811733384927114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,65536,0.044202665487925216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,5120,0.44008318583170575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,6144,0.5290687878926594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,7168,0.6242698669433594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,8192,0.6921599706013997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,3584,0.31524585088094076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,10240,0.8637813568115235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,4096,0.35505174001057943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,3072,0.2762506802876791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,2560,0.2314197381337484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,768,0.08476693630218506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,1024,0.10411307017008464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,12288,1.055841064453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,1536,0.14769280751546224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,2048,0.18852052688598633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,256,0.04370559851328532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,128,0.036294400691986084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,64,0.03650986750920614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,512,0.061222398281097413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,32,0.03171839912732442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,7168,0.15263999303181966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,8192,0.173415470123291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,10240,0.2179413318634033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,16384,1.3733525594075522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,12288,0.25985706647237145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,16384,0.3517258644104004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,5120,0.11088319619496663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,4096,0.09065173467000326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,6144,0.13168959617614745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,3072,0.06939413547515869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,3584,0.0833237330118815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,2560,0.06240426699320475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,512,0.0168938676516215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,768,0.022502400477727256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,256,0.011924266815185547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,1024,0.0284223993619283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,1536,0.0400330662727356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,2048,0.04833173354466756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,32,0.008077866832415263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,128,0.009553066889444987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,64,0.007561600208282471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,7168,0.11998079617818196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,8192,0.13539306322733563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,10240,0.1732927958170573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,12288,0.19989013671875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,16384,0.2683583895365397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,4096,0.07192746798197427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,5120,0.08677120208740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,6144,0.1036959966023763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,3584,0.06172266801198324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,2560,0.04669546683629354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,3072,0.05471573273340861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,1536,0.029572266340255737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,1024,0.02084160049756368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,2048,0.03760639826456706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,768,0.017056000232696534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,512,0.01322879989941915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,256,0.009410132964452107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,32,0.00658240020275116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,128,0.007257600128650665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,64,0.006369066735108693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,8192,0.11117973327636718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,10240,0.13744212786356608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,12288,0.1687189261118571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,16384,0.2183370590209961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,5120,0.07339519659678141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,7168,0.09847359657287598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,6144,0.08461973667144776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,4096,0.05841600100199381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,3584,0.05292906761169434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,3072,0.045475200812021895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,2560,0.03857066631317139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,1536,0.02650880018870036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,65536,1.383202107747396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,1024,0.018055466810862224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,2048,0.03207359910011291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,512,0.011896533767382304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,768,0.015014400084813436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,256,0.009305600325266521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,128,0.00698773314555486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,32,0.00647680014371872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,64,0.006289066871007283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,8192,0.0886677344640096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,10240,0.1120970646540324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,65536,1.0549493153889975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,12288,0.13033920129140217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,16384,0.1737823963165283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,6144,0.06903786659240722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,5120,0.05853013197580973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,7168,0.07828479607899984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,4096,0.04592959880828858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,3584,0.04161813259124756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,1024,0.014749866724014283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,1536,0.019859200716018675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,2048,0.025306665897369386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,2560,0.030705066521962483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,3072,0.03655786514282226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,768,0.012453333536783854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,512,0.010149332880973815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,32,0.0056991999348004665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,64,0.005383466680844625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,256,0.007366399963696797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,128,0.006137600044409434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,65536,0.8623082478841146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,8192,0.08175040086110433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,10240,0.10228053728739421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,12288,0.12463359832763672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,7168,0.07464959621429443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,16384,0.16195519765218097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,6144,0.05993493398030599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,4096,0.04228266477584839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,5120,0.055460266272226964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,3584,0.036933334668477376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,3072,0.03169493277867635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,768,0.010398933291435241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,1024,0.012742400169372559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,1536,0.017683200041453042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,2048,0.02257066567738851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,2560,0.026501333713531493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,512,0.008238933483759562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,256,0.006215466558933258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,64,0.004962133367856344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,128,0.005401599903901418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,32,0.0051242664456367494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,65536,0.6734879811604817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,8192,0.06800853411356608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,12288,0.10361920197804768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,10240,0.0853439966837565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,7168,0.06129599809646606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,16384,0.13409066200256348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,5120,0.045100800196329754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,4096,0.035997867584228516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,6144,0.05181866486867269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,2560,0.024257065852483113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,3584,0.03181973298390706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,3072,0.028074665864308672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,1024,0.011894399921099346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,2048,0.019655466079711914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,1536,0.01565439999103546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,768,0.010246400038401287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,512,0.00844693382581075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,256,0.0060597335298856105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,128,0.005226666728655497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,64,0.0046858668327331545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,32,0.004916266600290934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,65536,0.631929588317871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,8192,0.058082131544748936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,10240,0.07211519877115885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,7168,0.052944000562032065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,12288,0.086900266011556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,16384,0.1160096009572347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,3072,0.024995199839274087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,4096,0.031293867031733195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,3584,0.027849600712458296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,5120,0.03781333367029826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,2560,0.021337600549062093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,6144,0.04466453393300374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,768,0.009941333532333374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,2048,0.017461333672205606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,1024,0.011505066355069479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,1536,0.014299733440081277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,512,0.00788800021012624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,65536,0.5350570678710938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,256,0.0063391998410224915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,128,0.005076266825199127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,64,0.004710400104522705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,32,0.004828799764315287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,7168,0.04047573407491048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,12288,0.06893440087636313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,8192,0.045390931765238444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,10240,0.056411735216776525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,16384,0.0914400021235148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,3072,0.019042134284973145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,3584,0.021735467513402305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,5120,0.03054933349291484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,4096,0.02427306572596232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,6144,0.035276798407236735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,2560,0.016300800442695617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,1536,0.011335466305414836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,2048,0.013687466581662497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,65536,0.44840319951375324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,1024,0.009202133615811665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,768,0.007593599955240886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,256,0.005030400057633718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,512,0.006145066519578298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,128,0.0044608001907666525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,64,0.004092800120512644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,32,0.004195199906826019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,12288,0.060071468353271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,7168,0.035648000240325925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,8192,0.040676267941792806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,10240,0.055027198791503903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,16384,0.08040106296539307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,4096,0.021771732966105142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,3584,0.019236266613006592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,3072,0.017119999726613364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,5120,0.02637653350830078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,6144,0.0313973327477773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,65536,0.351197878519694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,1536,0.010403199990590414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,2560,0.014604799946149192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,2048,0.0123989333709081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,768,0.006817066669464111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,1024,0.008232533435026805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,256,0.004744533201058706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,512,0.005836800237496694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,128,0.00425493319829305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,32,0.003986133386691412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,64,0.00384853333234787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,8192,0.03583679993947347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,7168,0.03163520097732544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,10240,0.05285439888636271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,12288,0.05178240140279135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,16384,0.07010666529337564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,6144,0.027690666913986205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,5120,0.023840000232060753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,4096,0.019051732619603474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,2560,0.012988799810409546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,3584,0.01681173245112101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,3072,0.015086932977040609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,65536,0.30619306564331056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,2048,0.011075199643770853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,1024,0.007412266731262207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,1536,0.009474133451779682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,512,0.005373866856098175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,768,0.0063391998410224915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,256,0.004532266656557719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,64,0.0036789332826932275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,128,0.0040618665516376495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,32,0.0038624001046021783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,10240,0.04701120058695475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,8192,0.030876799424489336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,7168,0.032295467456181844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,12288,0.053233067194620766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,16384,0.057886934280395506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,6144,0.028513065973917645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,4096,0.016769067446390788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,65536,0.27186454137166344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,5120,0.024552534023920693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,3584,0.016851200660069784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,2560,0.012652800480524699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,3072,0.015502933661142984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,512,0.005022933085759481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,2048,0.010757333040237427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,768,0.005977599819501241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,1536,0.008762666583061218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,1024,0.006695466736952464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,128,0.0038719999293486277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,256,0.004321066538492838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,64,0.0035797332723935447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,32,0.003705599904060364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,12288,0.04981653292973836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,7168,0.03070613344510396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,10240,0.04459199905395508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,8192,0.03123413324356079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,16384,0.0523797353108724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,6144,0.025265065828959148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,65536,0.23582080205281578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,4096,0.018279467026392618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,5120,0.022412800788879396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,3584,0.016005333264668783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,3072,0.015180800358454385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,2560,0.012332800030708312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,2048,0.011363200346628825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,1536,0.008764800429344178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,1024,0.006743466854095459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,768,0.005568000177542368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,256,0.004084266722202301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,512,0.004691199958324432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,128,0.0036661334335803984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,64,0.003446399917205175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,32,0.0035157332817713416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,10240,0.038371201356252035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,8192,0.026370133956273394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,7168,0.025970133145650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,16384,0.042029865582784015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,12288,0.04305386543273926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,65536,0.17992533047993978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,6144,0.022707200050354003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,3584,0.013341866930325828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,4096,0.015588266650835672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,5120,0.020132267475128175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,3072,0.012839466333389282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,2048,0.009237333138783773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,2560,0.010995200276374817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,256,0.0037077332536379496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,1536,0.0073077330986658735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,1024,0.005773866673310598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,768,0.004994133114814758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,512,0.0042453333735466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,128,0.0033269333342711128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,64,0.0032746667663256326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,32,0.00329066663980484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,10240,0.025250132878621417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,12288,0.029155200719833373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,7168,0.019064533710479736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,8192,0.021422932545344033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,65536,0.14179840087890624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,16384,0.0315285325050354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,4096,0.011325866977373759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,3584,0.011283199985822041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,5120,0.013832533359527588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,6144,0.017450666427612303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,3072,0.009935999910036724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,2560,0.008218666911125183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,1536,0.005901866654555003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,2048,0.006569600105285645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,1024,0.004835199813048045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,768,0.0044064000248909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,512,0.003972266614437103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,256,0.0035413332283496858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,128,0.003357866654793421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,64,0.0031626666585604347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,32,0.003171200056870779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,65536,0.09769813219706217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,8192,0.01626240015029907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,10240,0.022963200012842813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,16384,0.026504532496134443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,12288,0.024987733364105223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,7168,0.014829867084821067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,6144,0.013230933745702108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,5120,0.01437226633230845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,4096,0.011143466830253601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,3584,0.009780266880989074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,3072,0.008552533388137818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,2560,0.007770666480064392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,1024,0.004637866715590159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,1536,0.006182399888833364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,2048,0.00687360018491745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,768,0.004167466859022776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,256,0.0033439998825391137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,512,0.0037717332442601522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,128,0.0031989333530267083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,64,0.0030410667260487873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,32,0.003052799900372823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,65536,0.0766048034032186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,8192,0.013643733660380044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,10240,0.018158932526906334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,12288,0.02030186653137207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,16384,0.0201365331808726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,7168,0.013863466183344521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,6144,0.01316159963607788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,5120,0.012155733505884806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,4096,0.009758933385213216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,3072,0.0077919999758402515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,3584,0.008881066242853801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,2560,0.007115733126799266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,1536,0.00561706672112147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,1024,0.004727466901143392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,2048,0.006414933502674103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,768,0.004265599946180979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,512,0.003832533210515976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,256,0.003311999887228012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,128,0.0032170665760835014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,64,0.0030048000315825146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,32,0.002996266633272171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,65536,0.05733760197957357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,8192,0.009103999535242716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,10240,0.011246933539708456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,16384,0.01344106694062551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,12288,0.012085333466529846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,6144,0.008840533097585042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,7168,0.010307199756304423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,5120,0.009113599856694538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,4096,0.008182399968306223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,3072,0.007669333120187123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,3584,0.007715199887752533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,2560,0.006355200211207073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,2048,0.005596800148487091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,1024,0.00425493319829305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,1536,0.004958933095137278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,768,0.003905066599448522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,256,0.0033013333876927694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,128,0.002963199963172277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,512,0.003573333223660787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,64,0.0029834667841593427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,65536,0.05067946513493856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,32,0.00290133332212766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,6144,0.008797867099444072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,7168,0.008050133287906647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,8192,0.008263466755549113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,10240,0.009845333298047383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,12288,0.009346133470535279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,16384,0.011010133226712545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,5120,0.008190933366616566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,3584,0.0072053333123524976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,4096,0.0075434664885203045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,2560,0.006218666831652323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,3072,0.006905599931875865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,2048,0.005539200206597646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,1024,0.004153600086768469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,1536,0.004839466512203216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,768,0.0038176000118255613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,512,0.0034656000634034475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,256,0.0031850665807724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,128,0.002937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,65536,0.04152533213297526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,64,0.0029333333174387617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,32,0.0030517332255840302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,6144,0.006588799754778545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,7168,0.006919466455777486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,10240,0.00765119989713033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,8192,0.007106133302052816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,12288,0.00823040008544922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,16384,0.009358933568000794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,5120,0.00702400008837382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,4096,0.006545066833496094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,3584,0.007044266661008198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,3072,0.006706133484840393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,2560,0.00610453337430954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,2048,0.0053727999329566956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,1536,0.004732800026734671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,1024,0.004063999901215235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,768,0.0037471999724706015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,65536,0.03394240140914917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,512,0.003369600077470144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,256,0.0030752000709374744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,128,0.002898133297761281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,64,0.0028490667541821797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,32,0.002811733384927114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,6144,0.0064746667941411335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,7168,0.006708266834417979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,8192,0.006829866766929626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,12288,0.007169066866238911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,16384,0.007901866734027863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,10240,0.00694400022427241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,5120,0.006849066913127899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,4096,0.006426666676998138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,3584,0.006857599814732869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,2560,0.006056533257166544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,3072,0.006594133377075195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,2048,0.005412266651789347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,1536,0.00476800004641215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,1024,0.0040554667512575785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,768,0.003697066754102707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,512,0.003370666752258936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,256,0.0030741333961486817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,128,0.002906666696071625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,65536,0.03362880150477092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,64,0.0028170667588710784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,32,0.0027903998891512555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,5120,0.35281492869059244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,6144,0.4122293472290039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,7168,0.47641814549763994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,8192,0.5356639862060547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,4096,0.28419841130574547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,3584,0.2471168041229248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,10240,0.6791584014892578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,3072,0.2135594685872396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,2560,0.18800959587097169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,1536,0.1207583983739217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,1024,0.08694612979888916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,2048,0.15038933753967285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,768,0.06957546869913736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,256,0.0391487995783488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,12288,0.8064042409261069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,64,0.027461334069569902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,128,0.030152533451716108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,512,0.050828798611958825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,32,0.028588799635569255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,7168,0.1214026689529419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,8192,0.13629013697306316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,10240,0.17117120424906412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,12288,0.2089183966318766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,16384,1.0623168309529623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,6144,0.1050602674484253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,5120,0.08727893034617105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,16384,0.276142946879069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,4096,0.07134719689687094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,3584,0.06476586659749349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,3072,0.057728000481923426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,768,0.017477333545684814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,256,0.009925333658854167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,2560,0.046854400634765626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,1024,0.021699200073877968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,1536,0.03188160061836243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,512,0.013381333152453104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,2048,0.03840106725692749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,128,0.0075445334116617845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,64,0.0065642664829889934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,32,0.0067071999112765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,7168,0.08921813170115153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,8192,0.10091520150502523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,10240,0.12756586869557698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,12288,0.14783573150634766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,4096,0.053607467810312906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,16384,0.19391039212544758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,5120,0.0651039997736613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,6144,0.07715840339660644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,3584,0.04666239817937215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,3072,0.041725865999857586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,2560,0.037115732828776046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,1024,0.01660586694876353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,1536,0.023296000560124715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,2048,0.02898026704788208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,768,0.013885866602261862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,512,0.011110400160153706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,32,0.006078933179378509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,256,0.008376533786455791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,128,0.0067104001839955645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,64,0.005820799867312113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,8192,0.08926400343577066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,10240,0.10915626684824627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,12288,0.1293503999710083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,16384,0.17602987289428712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,7168,0.07836053371429444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,6144,0.0682581345240275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,5120,0.0565888007481893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,4096,0.04777919848759969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,3584,0.041870931784311935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,2048,0.026445867617925008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,65536,1.0848373413085937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,2560,0.03126186728477478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,3072,0.03646293481190999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,1024,0.015007999539375306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,1536,0.02103360096613566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,768,0.013319466511408487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,128,0.006179200112819671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,256,0.007602133353551229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,512,0.01030293305714925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,64,0.005411200225353241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,32,0.005726933479309082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,8192,0.07038186391194662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,10240,0.0870901346206665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,65536,0.7718965530395507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,12288,0.10235839684804279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,16384,0.13580800692240397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,7168,0.06208106676737467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,5120,0.047126400470733645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,4096,0.036408531665802005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,6144,0.05349973440170288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,3072,0.02908693353335063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,3584,0.03280959924062093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,1536,0.015987199544906617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,1024,0.012101333340009053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,2560,0.024726400772730507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,2048,0.020057600736618043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,768,0.010300800204277039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,256,0.006284800171852112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,512,0.008449066678682964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,128,0.0053845331072807315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,64,0.004750933249791463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,32,0.004961066444714864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,65536,0.6697749455769857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,8192,0.06157653331756592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,10240,0.07861546675364176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,12288,0.0912928024927775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,7168,0.058032000064849855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,16384,0.12043200333913166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,6144,0.04753599961598714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,4096,0.03315946658452352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,5120,0.04028586546579997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,3072,0.026660267512003583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,3584,0.030822400252024335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,2048,0.018686934312184652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,1536,0.015064533551534018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,1024,0.011690666278203327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,2560,0.022875734170277915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,768,0.0102293332417806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,512,0.008594133456548055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,65536,0.5510933558146159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,128,0.005649066468079885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,256,0.006548266609509785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,64,0.00518506666024526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,32,0.005452799797058106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,7168,0.046470399697621664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,8192,0.0529802680015564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,10240,0.07358613014221191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,12288,0.07817920049031576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,16384,0.10828800201416015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,6144,0.040440531571706136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,5120,0.03811519940694173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,4096,0.02802773316701253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,3584,0.025486934185028075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,3072,0.022802132368087768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,2560,0.020092799266179403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,1536,0.013291733463605246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,768,0.00865600009759267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,2048,0.01685546636581421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,1024,0.009987200299898785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,65536,0.46840852101643876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,128,0.004702933132648468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,512,0.0071487997968991595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,256,0.005520000060399374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,64,0.004265599946180979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,32,0.004439466694990794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,7168,0.04101440111796061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,8192,0.0460586667060852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,10240,0.056591999530792234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,12288,0.06687040328979492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,16384,0.09137173493703207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,3072,0.02012053330739339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,6144,0.03590186834335327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,5120,0.030430932839711506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,4096,0.025176533063252765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,3584,0.02262293299039205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,2560,0.01685973405838013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,2048,0.013745066523551942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,512,0.0064640000462532045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,1536,0.011778133114178975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,1024,0.009165866176287334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,768,0.008055466910203297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,128,0.004499199986457825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,256,0.005052799979845682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,64,0.004217599829037985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,32,0.004379733403523763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,65536,0.4025589307149251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,7168,0.03849386771519979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,8192,0.04965333143870036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,12288,0.07277332941691081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,16384,0.09434239864349366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,10240,0.06717973550160726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,3584,0.01872640053431193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,3072,0.017181867361068727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,6144,0.03993173440297444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,4096,0.020435200134913126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,2560,0.014469333489735923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,5120,0.03414719899495443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,768,0.006664533416430156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,65536,0.35530026753743493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,2048,0.012131200234095255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,1536,0.01000853379567464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,1024,0.007805866499741872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,512,0.005275733272234599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,256,0.004457599918047587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,128,0.004005333284536997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,64,0.003705599904060364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,32,0.003802666564782461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,7168,0.03131306568781535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,10240,0.04348906675974528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,8192,0.03505920171737671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,12288,0.05363839864730835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,16384,0.06637866497039795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,5120,0.024225066105524697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,4096,0.01986560026804606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,3584,0.018309332927068076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,65536,0.271180788675944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,6144,0.029745066165924074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,3072,0.016173866391181946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,2560,0.012811733285586038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,1536,0.011367467045783997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,2048,0.011024000247319539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,768,0.0073183998465538025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,1024,0.00897173285484314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,512,0.005782400071620941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,256,0.004645333190759023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,128,0.004084266722202301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,64,0.0037717332442601522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,32,0.003953066716591517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,10240,0.03793066740036011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,12288,0.04426133235295614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,7168,0.028230400880177815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,8192,0.031531733274459836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,16384,0.05936959981918335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,5120,0.0204202671845754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,4096,0.019001599152882895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,65536,0.24570345878601074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,3584,0.017118932803471883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,6144,0.02542399962743123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,2560,0.012745599945386252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,3072,0.013648000359535218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,768,0.006141866743564606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,2048,0.009846400221188862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,1024,0.006618666648864746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,1536,0.008933333555857341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,512,0.005089066425959269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,256,0.004195199906826019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,128,0.003718400001525879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,32,0.0036960000793139136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,64,0.00352960005402565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,10240,0.03334720134735107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,7168,0.024168533086776734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,12288,0.039773865540822344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,16384,0.053965866565704346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,8192,0.026702932516733807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,3584,0.014254933595657349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,6144,0.021831466754277547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,65536,0.2061247984568278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,5120,0.018966400623321535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,4096,0.014376533031463624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,3072,0.011403733491897583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,2560,0.010948266585667927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,2048,0.009616000453631084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,1536,0.008225066463152568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,1024,0.00581226646900177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,768,0.005283200244108836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,512,0.004533333579699198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,256,0.003972266614437103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,128,0.0035413332283496858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,64,0.003365333378314972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,32,0.0034634667138258614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,7168,0.028880000114440918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,10240,0.03983253240585327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,12288,0.04586879809697469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,8192,0.031890134016672775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,65536,0.1820192019144694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,16384,0.047966933250427245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,5120,0.02035306692123413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,6144,0.025832533836364746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,4096,0.014325333635012307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,3584,0.013091199596722922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,3072,0.01130133370558421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,2560,0.010809600353240967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,2048,0.008819199601809184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,1024,0.005614933371543884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,1536,0.007537066439787547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,256,0.0038261334101359046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,768,0.004856533308823904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,512,0.004271999994913737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,64,0.0032960000137488045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,128,0.00345920001467069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,32,0.0033632000287373864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,10240,0.025569067398707075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,8192,0.02095466653505961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,7168,0.018678400913874307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,12288,0.029890133937199907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,65536,0.14310399691263836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,16384,0.036374398072560626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,6144,0.0165503998597463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,5120,0.014325333635012307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,4096,0.012086400389671325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,3584,0.010621866583824158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,2048,0.00876693328221639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,1536,0.007192533214886982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,3072,0.010101333260536194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,2560,0.009825066725413004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,1024,0.005430399874846141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,256,0.003656533360481262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,512,0.00415040006240209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,768,0.004791466891765595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,128,0.003369600077470144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,32,0.0032000000278155005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,64,0.003189333279927572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,16384,0.029953066507975263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,12288,0.02799786726633708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,8192,0.016457600394884746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,65536,0.13560214042663574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,7168,0.014984533190727234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,10240,0.025041067600250246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,6144,0.013316266735394797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,4096,0.010075733065605164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,3584,0.009296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,3072,0.008521599570910136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,5120,0.011869866649309795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,2560,0.007978666822115581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,2048,0.006749866902828217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,128,0.003218133250872294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,1536,0.0056757330894470215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,1024,0.004859733581542969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,768,0.004211199780305227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,256,0.0034346667428811393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,512,0.003853866706291834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,64,0.003083733220895131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,65536,0.1029205322265625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,32,0.003218133250872294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,10240,0.019242666165033975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,12288,0.020730666319529214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,8192,0.016011733810106912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,16384,0.025113600492477416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,7168,0.014761599898338317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,6144,0.012999467055002847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,3584,0.009096533060073853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,5120,0.0115146666765213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,4096,0.009864532947540283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,2560,0.007214933137098948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,3072,0.008141866823037466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,2048,0.0062826668222745255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,512,0.003755733370780945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,1536,0.005543466905752818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,1024,0.004654933512210846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,768,0.004243200023969015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,256,0.0033226666351159418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,128,0.003068800022204717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,64,0.003054933249950409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,65536,0.06760853131612142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,32,0.003102933367093404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,8192,0.012688000003496805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,6144,0.011994666854540507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,16384,0.017386666933695474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,10240,0.014424533645311991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,12288,0.015842133760452272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,7168,0.011985066533088683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,5120,0.010568533341089885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,4096,0.00888213316599528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,3072,0.007045333087444305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,3584,0.007819733520348867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,2048,0.0064416001240412395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,2560,0.007315200070540111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,1024,0.004593066871166229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,1536,0.005765333275000254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,768,0.004171733558177948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,256,0.0032287999987602235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,512,0.003789866715669632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,128,0.003011200080315272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,65536,0.0515882651011149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,64,0.002998399982849757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,32,0.002996266633272171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,6144,0.008611200253168742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,10240,0.010934399565060933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,7168,0.009244799613952637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,8192,0.009507200121879578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,16384,0.011619200309117634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,12288,0.012346667051315308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,3584,0.007626666625340779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,5120,0.008372267087300617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,4096,0.007704533139864604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,3072,0.006908800204594929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,2560,0.006247466802597046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,2048,0.005547733108202616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,1536,0.004853333532810211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,1024,0.00415786678592364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,768,0.00377813329299291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,65536,0.04526506662368775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,512,0.003469866762558619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,256,0.003124266614516576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,128,0.002942933390537898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,64,0.002898133297761281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,32,0.0028629332780838014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,6144,0.007477333148320516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,7168,0.008008533219496409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,12288,0.010270933310190838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,8192,0.00846506655216217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,10240,0.00939520001411438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,16384,0.012640000383059184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,5120,0.007095466554164887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,4096,0.006673066814740498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,2560,0.006237866481145223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,3072,0.006849066913127899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,2048,0.005533866584300995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,3584,0.007163733243942261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,1536,0.004851200183232625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,65536,0.03644160032272339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,1024,0.004183466732501984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,768,0.0037994667887687682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,256,0.0031338666876157125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,512,0.003472000112136205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,128,0.0029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,64,0.0028373333315054577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,32,0.0028575999041398365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,7168,0.007212799787521362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,10240,0.008337066570917765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,8192,0.0075989335775375364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,12288,0.008994133273760477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,16384,0.010275200009346008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,6144,0.006549333532651265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,5120,0.006968533496061961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,1536,0.004722133278846741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,3072,0.006714666883150737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,3584,0.006994133194287618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,4096,0.00664213349421819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,2560,0.0060586666067441305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,2048,0.005385600030422211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,65536,0.029410133759180706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,1024,0.004082133372624716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,768,0.003703466554482778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,256,0.003031466652949651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,512,0.0033791999022165934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,32,0.0027978666126728057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,64,0.0027477333943049112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,128,0.002918400118748347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,8192,0.006874666611353557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,7168,0.006586666901906331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,10240,0.007201066613197327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,6144,0.006483200192451477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,16384,0.008549333612124125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,12288,0.007855999966462452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,5120,0.006797866523265838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,3584,0.007008000214894612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,1536,0.004710400104522705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,2560,0.006061866879463196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,4096,0.006413866579532623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,3072,0.006559999783833821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,2048,0.005410133302211762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,65536,0.027398399511973065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,768,0.0036960000793139136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,1024,0.004082133372624716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,32,0.0027989332874615988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,512,0.0033589333295822145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,256,0.0029781334102153777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,128,0.0028959999481836954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,64,0.0027615999182065325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,5120,0.2578069368998209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,6144,0.30695466995239257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,7168,0.35286614100138347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,8192,0.39736639658610023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,10240,0.4994997342427571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,4096,0.21038293838500977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,3584,0.18928213119506837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,3072,0.1668384075164795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,2560,0.15519679387410482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,1024,0.07162559827168782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,12288,0.5984085083007813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,768,0.056807466348012293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,1536,0.09575680096944174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,2048,0.1268831968307495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,512,0.04034133354822795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,128,0.024472532669703166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,256,0.02924373348553975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,32,0.016775466998418174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,64,0.01616320013999939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,16384,0.7734538396199544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,7168,0.09143786430358887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,8192,0.1026911973953247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,10240,0.12670186360677083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,12288,0.15254079500834147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,6144,0.09050347010294596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,4096,0.055326934655507407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,5120,0.06756160259246827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,16384,0.19620906511942546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,3584,0.04992640018463135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,3072,0.04671466747919718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,1024,0.020067199071248373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,2560,0.03790719906489055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,1536,0.02577600081761678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,768,0.014260266224543253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,2048,0.03153280019760132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,512,0.010981333255767823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,128,0.006132266422112783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,256,0.008451199531555176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,64,0.005504000186920166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,32,0.005804799993832906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,7168,0.06861440340677896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,8192,0.07621973355611165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,12288,0.11347306569417318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,10240,0.09457066853841146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,16384,0.14683094024658203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,4096,0.04212693373362224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,3584,0.03803733189900716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,5120,0.05078826745351156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,6144,0.059740801652272545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,1024,0.013381333152453104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,3072,0.03683626651763916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,2560,0.02899199922879537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,1536,0.019436800479888917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,2048,0.024064000447591147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,256,0.006785066425800323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,768,0.01111466685930888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,512,0.009136000275611877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,32,0.005094400048255921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,64,0.004620799918969473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,128,0.005130666494369507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,8192,0.06650133530298868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,10240,0.08181333541870117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,12288,0.09763200283050537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,16384,0.12527360121409098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,5120,0.044334932168324785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,7168,0.05885226726531982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,6144,0.051837865511576334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,4096,0.038328532377878824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,3584,0.03335893154144287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,2560,0.025550933678944905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,3072,0.02959146698315938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,1536,0.017035732666651406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,2048,0.0213536004225413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,65536,0.7715434392293294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,1024,0.012296533584594727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,512,0.008904533584912618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,768,0.010634666681289673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,256,0.006405333181222279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,32,0.0049674664934476215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,64,0.004647466540336609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,128,0.005178666611512502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,8192,0.05367679993311564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,10240,0.06632639964421591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,12288,0.07848320007324219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,65536,0.5646165211995442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,16384,0.10123946666717529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,5120,0.03628693421681722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,7168,0.04786026477813721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,6144,0.042062934239705405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,4096,0.03325759967168172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,3584,0.02679786682128906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,3072,0.02388906677563985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,2560,0.02067199945449829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,1536,0.014533332983652749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,2048,0.016846932967503867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,1024,0.009894399841626485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,768,0.008521599570910136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,512,0.007159466544787089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,128,0.0045621335506439206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,256,0.00518506666024526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,64,0.0041002665956815084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,32,0.004422399898370107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,65536,0.5303178787231445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,8192,0.0466815988222758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,10240,0.060813868045806886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,12288,0.06844159762064615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,7168,0.04766826629638672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,16384,0.08838826815287272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,6144,0.03758613268534343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,5120,0.03307626644770305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,2560,0.018257067600886027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,3584,0.02372693419456482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,4096,0.02656000057856242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,3072,0.021861332654953002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,1536,0.011476266384124755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,2048,0.014465066790580749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,1024,0.009868799646695455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,768,0.00786240001519521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,512,0.006471466521422069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,256,0.004923733572165171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,128,0.004294399917125702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,32,0.004117333392302195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,64,0.0038634667793909705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,65536,0.40131413141886396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,7168,0.04232213497161865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,8192,0.041670401891072587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,16384,0.0781162659327189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,10240,0.054767998059590664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,12288,0.06081706682840983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,4096,0.023575466871261597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,5120,0.029649066925048827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,3072,0.01949119965235392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,3584,0.02113599975903829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,6144,0.03353493213653565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,2560,0.016051200032234193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,512,0.005667200187842051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,2048,0.01309653321901957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,1536,0.010521599650382995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,1024,0.008184533317883809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,768,0.007462400197982788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,65536,0.34486614863077797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,256,0.004636799792448679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,128,0.004110933343569437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,64,0.0037632000943024955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,32,0.003984000037113826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,12288,0.05757866700490316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,8192,0.036508798599243164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,7168,0.034569601217905685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,10240,0.055468801657358804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,16384,0.07714347044626871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,3072,0.01750826636950175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,4096,0.021283199389775596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,6144,0.03136853377024333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,5120,0.02812160054842631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,3584,0.0192522664864858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,2560,0.014778666694959006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,1536,0.01048533320426941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,2048,0.012232533097267151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,768,0.00693333347638448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,1024,0.007749333480993907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,65536,0.30082880655924477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,512,0.005321600039800008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,256,0.0044608001907666525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,128,0.003931733220815659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,64,0.003585066646337509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,32,0.0038975998759269714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,7168,0.031675734122594196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,12288,0.04830933411916097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,8192,0.034040534496307374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,10240,0.04505493243535359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,16384,0.06194133361180624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,3072,0.014709333578745524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,3584,0.016331733266512553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,5120,0.02358506719271342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,4096,0.01919040083885193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,6144,0.02645546595255534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,2560,0.012685867150624594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,65536,0.27949654261271156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,2048,0.010687999924023946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,1536,0.008890666564305623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,512,0.004738133152325948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,768,0.006093866626421611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,1024,0.007403733332951863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,256,0.004070399949947992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,128,0.0035989334185918174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,32,0.0035968000690142312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,64,0.0034282666941483817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,8192,0.029372799396514892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,7168,0.026539733012517292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,10240,0.03566293319066365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,12288,0.045961598555246994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,16384,0.055410134792327884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,5120,0.02016106645266215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,6144,0.023513599236806234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,65536,0.2141109307607015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,4096,0.01771519978841146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,3584,0.014697600404421488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,3072,0.013853866855303446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,2560,0.011763200163841248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,2048,0.010315733154614766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,1536,0.008684800068537394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,1024,0.00652159998814265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,768,0.005308799942334493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,512,0.004507733384768168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,256,0.0038442666331926978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,128,0.0034495999415715536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,64,0.0032416000962257386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,32,0.0034389334420363107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,10240,0.03657173315684001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,12288,0.041682132085164386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,8192,0.027323732773462932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,7168,0.024950400988260905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,16384,0.04946133295694987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,5120,0.018895999590555827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,65536,0.1869770685831706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,6144,0.021805866559346517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,4096,0.0151146670182546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,3584,0.014040533701578775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,3072,0.01262933313846588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,2560,0.010867200295130412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,1536,0.007841066519419352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,2048,0.00962666670481364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,1024,0.005964800218741099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,768,0.004884266853332519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,256,0.003668266783157984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,512,0.004287999868392944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,128,0.0033002667129039764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,32,0.0033258666594823206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,64,0.003235200047492981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,10240,0.03299733400344849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,7168,0.020552533864974975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,12288,0.03598293463389079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,8192,0.022641066710154215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,16384,0.04303573369979859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,65536,0.16072640419006348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,6144,0.017922133207321167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,3072,0.011762133240699768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,3584,0.012708266576131185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,5120,0.017465599377950034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,4096,0.013834666212399802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,2560,0.01007466713587443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,1536,0.007309866448243459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,2048,0.008694400389989216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,1024,0.005264000097910563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,768,0.004644266764322917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,512,0.004110933343569437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,128,0.0033344000577926634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,256,0.0034965333839257562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,64,0.0031776001056035364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,32,0.0032416000962257386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,8192,0.022728532552719116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,7168,0.020615466435750327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,12288,0.03411306540171306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,10240,0.030767999092737836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,16384,0.036899201075236004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,65536,0.1391306718190511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,4096,0.010686933000882467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,6144,0.018158932526906334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,5120,0.016504533092180886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,3584,0.009955199559529622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,3072,0.010135466853777569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,2560,0.00862506628036499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,2048,0.00713919997215271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,1536,0.006205866734186808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,1024,0.004874666531880697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,128,0.003352533280849457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,768,0.004341333111127218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,512,0.004036266605059306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,64,0.00317546675602595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,256,0.0035594666997591654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,32,0.0031989333530267083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,10240,0.024436267217000325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,65536,0.11317120393117268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,8192,0.017237333456675212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,7168,0.015264000495274863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,12288,0.027306665976842243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,6144,0.013634133338928222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,16384,0.030261333783467608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,5120,0.014697600404421488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,3584,0.010024533669153849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,2560,0.007778133451938629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,4096,0.010806399583816528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,3072,0.009645866354306538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,2048,0.006826666494210561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,768,0.0042015999555587765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,1024,0.0049898669123649595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,1536,0.005931733548641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,512,0.0037461332976818085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,64,0.003050666550795237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,256,0.003432533393303553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,128,0.0030432000756263735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,32,0.0031210665901501974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,65536,0.08946773211161295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,12288,0.020742400487263998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,8192,0.013537066181500754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,10240,0.01935466726620992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,16384,0.025526400407155352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,7168,0.013330133756001792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,5120,0.013075199723243714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,6144,0.014300800363222756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,4096,0.010616532961527507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,3584,0.01011840005715688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,2560,0.007208533088366191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,3072,0.00878613293170929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,2048,0.006843733290831248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,1536,0.005588266750176748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,128,0.0031317333380381264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,1024,0.004690133531888326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,768,0.0041792000333468115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,512,0.0036906667053699495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,256,0.0032629333436489105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,64,0.003005866706371307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,65536,0.06543360153834024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,32,0.0031114667654037476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,12288,0.014735999703407287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,8192,0.012776533762613932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,6144,0.012100266416867574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,16384,0.01850773294766744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,7168,0.01221226652463277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,10240,0.014588800072669984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,3584,0.008308266599973042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,5120,0.010713600118954976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,4096,0.00921493371327718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,2560,0.006720000008742015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,3072,0.007065600156784058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,2048,0.005791999896367391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,1536,0.005184000233809153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,768,0.004041599979003271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,1024,0.0044266665975252785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,512,0.0036789332826932275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,256,0.003256533294916153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,128,0.0029845332105954488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,64,0.0029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,65536,0.053666134675343834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,32,0.0029909332593282064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,8192,0.009752532839775086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,10240,0.010466133554776508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,7168,0.011609599987665812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,12288,0.012219732999801636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,6144,0.00909440020720164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,16384,0.013063466548919678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,2560,0.006372266511122386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,5120,0.009343999624252319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,3584,0.007516799867153168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,2048,0.0056320001681645715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,3072,0.007516799867153168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,4096,0.008384000261624653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,1536,0.004973866542180379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,65536,0.050145065784454344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,1024,0.0042453333735466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,512,0.0035904000202814737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,768,0.003886933376391729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,256,0.003173333406448364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,128,0.0030069333811601003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,64,0.0029098667204380036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,32,0.0028917332490285236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,7168,0.007448533177375793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,16384,0.011508267124493916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,12288,0.010050132870674133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,10240,0.010218666990598042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,8192,0.007850666840871174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,6144,0.008869333068529765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,5120,0.008200533191363017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,4096,0.007605333129564922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,1536,0.004820266862710317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,3584,0.007197866837183635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,3072,0.006877866884072621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,2560,0.006248533229033152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,2048,0.005530666808287303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,65536,0.037302398681640626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,768,0.0038058665891488397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,1024,0.004193066557248434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,512,0.0034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,256,0.003202133377393087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,32,0.0030623999734719592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,128,0.002985599885384242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,64,0.0027797333896160126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,8192,0.007117866476376851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,6144,0.006660266717274983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,7168,0.006902400155862172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,10240,0.007669333120187123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,16384,0.0100000003973643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,12288,0.00864746669928233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,5120,0.007101866602897644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,65536,0.02601813276608785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,4096,0.006696533163388569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,1536,0.004733866453170777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,3584,0.00709440012772878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,3072,0.006786133348941803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,2560,0.006085333228111267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,2048,0.005338666836420695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,512,0.0033749334514141084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,1024,0.004063999901215235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,768,0.003676799933115641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,256,0.0030879999200503034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,64,0.002791466563940048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,128,0.002942933390537898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,32,0.0027893332143624624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,16384,0.007423999905586243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,12288,0.00681279997030894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,10240,0.006939733525117238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,6144,0.006551466882228851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,8192,0.006885333359241486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,7168,0.006759466727574666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,5120,0.00689386675755183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,4096,0.006494933366775512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,65536,0.021705599625905354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,1536,0.004711466530958811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,3584,0.007011199990908305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,3072,0.006649599969387054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,2560,0.006033066908518473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,2048,0.005371733506520589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,1024,0.004011733333269755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,512,0.003421866645415624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,768,0.0036864000062147772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,256,0.0030261332790056865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,128,0.0028607999285062153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,64,0.002784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,32,0.00275093341867129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,16384,0.006723199784755707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,12288,0.006646400193373363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,8192,0.006759466727574666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,6144,0.006392533580462138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,7168,0.006614399949709575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,10240,0.0068256000677744556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,5120,0.006859733164310456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,4096,0.006344533463319142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,65536,0.014971733093261719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,3584,0.006855466465155284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,3072,0.006532266736030579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,2560,0.006065066655476888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,2048,0.005371733506520589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,1536,0.004728533327579498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,1024,0.004053333401679992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,768,0.003681066632270813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,32,0.0027903998891512555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,512,0.0033333333830038703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,256,0.0030773334205150605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,128,0.002890666574239731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,64,0.0027637332677841187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,5120,0.2465514659881592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,6144,0.2919424057006836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,7168,0.33773972193400065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,8192,0.3794816017150879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,10240,0.48030827840169266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,4096,0.2018954594930013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,3584,0.17883520126342772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,3072,0.16186347007751464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,2560,0.1500725269317627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,2048,0.12392106850941975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,1024,0.07082986831665039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,768,0.056731732686360684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,12288,0.5669877370198567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,512,0.04010453224182129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,1536,0.09339413642883301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,256,0.026140799125035603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,128,0.018578133980433145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,64,0.014328533411026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,32,0.015177599589029946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,7168,0.08547200361887614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,16384,0.7427050908406575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,8192,0.09731626510620117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,10240,0.12024426460266113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,12288,0.14242666562398273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,3072,0.045746131738026934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,4096,0.05297600030899048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,3584,0.046931199232737225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,5120,0.0728117307027181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,16384,0.1852981408437093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,6144,0.07529599666595459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,512,0.00997440020243327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,1024,0.01681813398996989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,768,0.014503467082977294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,1536,0.02407360076904297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,2048,0.029951999584833782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,2560,0.03588266770044963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,128,0.005562666555245717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,256,0.007401599983374278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,64,0.004830933113892873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,32,0.005275733272234599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,7168,0.06562666495641073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,8192,0.07368960380554199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,10240,0.09202346801757813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,12288,0.10867520173390706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,16384,0.1408799966176351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,6144,0.057966931660970056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,4096,0.04043200016021729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,3584,0.03623466491699219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,5120,0.04908373355865479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,3072,0.03514133294423421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,1024,0.012447999914487202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,1536,0.01837973395983378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,2048,0.022934399048487344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,2560,0.027803732951482134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,768,0.01055999994277954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,256,0.0056639999151229855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,512,0.008289066453774769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,128,0.004626133541266123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,32,0.0045525332291920986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,64,0.004240000247955322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,8192,0.06575359900792441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,10240,0.08846186796824138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,12288,0.10482559998830158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,16384,0.13873066902160644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,6144,0.05055786768595377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,7168,0.05835946798324585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,5120,0.04291520118713379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,4096,0.037732267379760744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,3584,0.03571199973424276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,1536,0.016301866372426352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,3072,0.028514132897059126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,2560,0.024358399709065757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,2048,0.020348799228668214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,1024,0.01244160036245982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,768,0.009704533219337463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,512,0.0075093333919843035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,128,0.004626133541266123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,256,0.005309866865475973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,65536,0.7144725163777669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,32,0.0046165332198143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,64,0.004124800115823746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,65536,0.5774965286254883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,8192,0.051658666133880614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,10240,0.06713493665059408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,12288,0.08883626461029052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,16384,0.09868799845377604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,6144,0.04197653134663899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,7168,0.053364264965057376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,5120,0.03588693141937256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,4096,0.03171520034472148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,3584,0.02693333427111308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,1536,0.012201600273450216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,2048,0.015735466281572977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,1024,0.009392000238100688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,2560,0.02176426649093628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,3072,0.022538665930430093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,768,0.008022400240103405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,256,0.004452266792456309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,512,0.006683733562628429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,65536,0.47959359486897785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,128,0.003942399968703588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,64,0.0037461332976818085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,32,0.004057600100835165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,8192,0.044880000750223796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,12288,0.07685759862263998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,7168,0.040491731961568196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,10240,0.06039679845174154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,16384,0.1064138650894165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,5120,0.03090133269627889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,3584,0.0229312002658844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,4096,0.025382399559020996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,6144,0.03611413240432739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,3072,0.020708266894022623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,2560,0.017548799514770508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,1024,0.008855467041333515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,1536,0.011363200346628825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,2048,0.01439786652723948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,512,0.006265600025653839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,65536,0.4033962567647298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,768,0.007765333354473114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,256,0.004692266881465912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,64,0.003866666555404663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,32,0.004158933212359746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,128,0.0042463997999827065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,7168,0.03519359827041626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,8192,0.039943468570709226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,10240,0.06512213150660197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,12288,0.05757226546605428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,16384,0.08442560036977133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,5120,0.031592533985773726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,6144,0.033745066324869795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,4096,0.022525866826375328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,2560,0.015100799997647605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,3584,0.020230400562286376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,3072,0.019454934199651084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,1024,0.007970133423805236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,2048,0.01222933332125346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,1536,0.010500267148017883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,768,0.006906666855017345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,256,0.004204800228277842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,512,0.005454933146635691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,128,0.0037493333220481873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,32,0.0037439999481042228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,64,0.003549866626660029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,65536,0.3483637491861979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,8192,0.03935466607411702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,7168,0.03629973332087199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,10240,0.04308693408966065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,12288,0.0511626680692037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,16384,0.06553599834442139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,3072,0.015726932883262636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,5120,0.02371413310368856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,4096,0.01953386664390564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,6144,0.030362667640050252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,3584,0.019268266359965005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,65536,0.29891840616861975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,512,0.005356800059477488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,2560,0.01330880026022593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,1536,0.009239466985066731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,2048,0.012478933731714884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,1024,0.007216000060240428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,768,0.006389333307743073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,256,0.0040394666294256846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,128,0.0037952000896135964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,64,0.0034677334129810332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,32,0.003533866753180822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,12288,0.05031359990437826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,7168,0.03078506588935852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,8192,0.031037867069244385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,10240,0.0521781325340271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,16384,0.06102933486302694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,6144,0.02794666687647502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,3584,0.014758400122324624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,65536,0.2440863927205404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,3072,0.014239999651908874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,4096,0.016685867309570314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,5120,0.021715199947357176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,2560,0.01160426636536916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,1536,0.008617599805196125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,2048,0.009865599870681762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,1024,0.0064074665307998655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,768,0.00558186670144399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,512,0.004540800054868063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,256,0.0039264000952243805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,128,0.0035349334279696143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,64,0.0032543999453385672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,32,0.0034858666360378264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,10240,0.037905065218607585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,8192,0.026232532660166424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,12288,0.04617600043614705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,7168,0.023727999130884806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,16384,0.04810880025227864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,4096,0.01530346671740214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,65536,0.20114879608154296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,3584,0.013639466961224875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,6144,0.021211733420689903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,5120,0.020721065998077392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,2560,0.012147200107574464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,3072,0.014906666676203408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,2048,0.010565333565076192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,512,0.004571733375390371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,1536,0.008637866377830506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,1024,0.006651733318964641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,768,0.0053727999329566956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,128,0.0035786665976047516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,256,0.0038751999537150065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,64,0.0033301333586374915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,32,0.0035135999321937563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,8192,0.027475200096766156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,10240,0.02872106631596883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,7168,0.0271722674369812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,12288,0.04624533255894979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,16384,0.0458901325861613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,6144,0.01889066696166992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,65536,0.17688533465067546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,5120,0.01617173353830973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,4096,0.013109333316485085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,3072,0.014029866456985474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,3584,0.013700266679128012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,2560,0.011528533697128297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,2048,0.009100799759229023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,256,0.0037578667203585305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,768,0.005120000243186951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,1024,0.006027733286221823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,1536,0.00798826664686203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,512,0.004420266548792521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,64,0.003323733309904734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,128,0.0034730667869249977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,32,0.003435733417669932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,10240,0.028938666979471846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,8192,0.02055786649386088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,12288,0.033508265018463136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,65536,0.15372907320658366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,7168,0.021011199553807577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,16384,0.043025068442026776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,6144,0.018549333016077675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,3584,0.011757866541544596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,3072,0.009477333227793375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,4096,0.013381333152453104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,5120,0.013992533087730408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,2560,0.009867733716964722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,2048,0.00840106705824534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,1536,0.006427733103434245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,1024,0.005333333214124044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,768,0.004315733412901561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,512,0.003927466770013174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,64,0.0031328000128269195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,256,0.0035914666950702667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,128,0.003337600082159042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,32,0.0030623999734719592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,8192,0.0189301331837972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,7168,0.017179733514785765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,12288,0.03260800043741862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,65536,0.1351637363433838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,10240,0.02844053308169047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,16384,0.03926080067952474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,6144,0.01520853340625763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,4096,0.014547200004259745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,5120,0.017667200167973837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,3584,0.012162133057912191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,2560,0.010200533270835876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,3072,0.011730133493741354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,1536,0.006106666723887126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,2048,0.008475733796755473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,1024,0.004692266881465912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,768,0.004253866771856943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,512,0.0038751999537150065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,256,0.003398400048414866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,128,0.003209600100914637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,64,0.0030432000756263735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,32,0.0031658666829268134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,10240,0.021782400210698445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,8192,0.01819093426068624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,65536,0.11088639895121258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,7168,0.01471573313077291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,12288,0.02439893285433451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,16384,0.029499733448028566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,5120,0.01288746694723765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,6144,0.014548266927401224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,4096,0.010934399565060933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,3584,0.010045866171518963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,2048,0.007155199845631917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,3072,0.009142399827639262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,1536,0.005785599847634633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,2560,0.007715199887752533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,768,0.004244266450405121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,1024,0.0047541335225105286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,512,0.00377813329299291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,128,0.003197866678237915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,256,0.003366400053103765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,64,0.0030613332986831666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,32,0.0029909332593282064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,65536,0.08996586799621582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,12288,0.018807466824849448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,8192,0.013767466942469279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,7168,0.012863999605178833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,16384,0.023625600337982177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,10240,0.01602026621500651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,5120,0.010634666681289673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,6144,0.01209386686484019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,2048,0.005952000121275584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,3584,0.008453333377838134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,4096,0.009270399808883667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,3072,0.007785599927107494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,2560,0.0070592001080513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,1536,0.005195733408133189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,1024,0.004394666850566864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,768,0.00403413325548172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,65536,0.06272746721903483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,256,0.003339733431736628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,512,0.0036373332142829893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,128,0.0029898665845394133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,32,0.0029696000119050344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,64,0.0029343999922275543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,7168,0.013522133231163025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,12288,0.017128533124923705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,10240,0.01551253298918406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,8192,0.01350933313369751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,16384,0.017051732540130614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,6144,0.012161067128181458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,5120,0.01069546639919281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,4096,0.00927786628405253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,3584,0.008331733445326488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,3072,0.0076330666740735366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,2048,0.005780266722043356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,2560,0.0064298664530118305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,1536,0.005043200155099233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,1024,0.004461866617202759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,65536,0.05013653437296549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,768,0.004130133241415024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,256,0.0032629333436489105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,512,0.003718400001525879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,128,0.0029663999875386557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,64,0.002958933264017105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,32,0.002833066632350286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,6144,0.009540266791979472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,7168,0.010373333096504211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,12288,0.013383466998736063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,10240,0.012210133671760558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,8192,0.010819199681282043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,16384,0.012875733772913614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,5120,0.009141332904497783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,4096,0.007890133559703827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,3584,0.00764160007238388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,2560,0.006276266773541768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,3072,0.006896000107129414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,2048,0.00555626650651296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,65536,0.038865065574645995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,1536,0.004839466512203216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,1024,0.004170666635036469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,768,0.0038250667353471124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,32,0.0028384000062942503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,512,0.003469866762558619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,256,0.003125333289305369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,128,0.002963199963172277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,64,0.0028170667588710784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,12288,0.011402666568756104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,10240,0.010067199667294819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,8192,0.008898133039474487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,7168,0.008516266942024231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,6144,0.007681066791216533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,16384,0.013801599542299906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,5120,0.0072618668278058365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,65536,0.034790400664011636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,1536,0.00483840008576711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,4096,0.006790400048096975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,3584,0.007166933516661327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,2560,0.0062389334042867025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,3072,0.006917333106199901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,2048,0.005527466535568237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,768,0.00378560001651446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,1024,0.0041802664597829185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,512,0.0034495999415715536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,128,0.0028917332490285236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,256,0.003068800022204717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,64,0.002850133428970973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,32,0.0028362666567166646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,16384,0.009169066945711773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,10240,0.00745066652695338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,7168,0.006921599805355072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,8192,0.007066666583220164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,6144,0.00660693347454071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,12288,0.007864533364772797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,5120,0.006986666719118755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,65536,0.025730133056640625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,1536,0.004726399978001913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,4096,0.006638933221499126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,2048,0.005386666456858317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,3584,0.007106133302052816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,2560,0.006041599810123444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,3072,0.006742399930953979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,1024,0.004082133372624716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,768,0.003697066754102707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,512,0.0033439998825391137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,256,0.0030741333961486817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,128,0.0028959999481836954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,64,0.00277866671482722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,32,0.002757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,16384,0.007606400052706401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,10240,0.006916266679763794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,8192,0.006911999980608622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,7168,0.006715733309586842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,12288,0.00698773314555486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,5120,0.006867200136184692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,65536,0.014173866311709086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,6144,0.006493866443634033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,2048,0.005389866729577383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,1536,0.0047189335028330484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,3584,0.006983466446399689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,4096,0.006513066589832306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,3072,0.006631466746330261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,2560,0.006025599936644236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,1024,0.004031999905904134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,768,0.003692800054947535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,512,0.0033344000577926634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,64,0.0027434666951497394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,32,0.0026634665826956432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,128,0.0029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,256,0.0031680000325044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,16384,0.0067775999506314594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,12288,0.006681600213050842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,10240,0.0068458666404088335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,8192,0.006773333251476288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,7168,0.006634666522343953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,65536,0.013384532928466798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,4096,0.006388266881306966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,6144,0.0064416001240412395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,5120,0.006876799960931141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,3584,0.006851199766000111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,3072,0.006546133259932201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,1536,0.004667733112970988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,2560,0.006001066664854685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,2048,0.005362133185068766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,768,0.003626666714747747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,1024,0.0040287998815377556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,512,0.003345066557327906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,128,0.0028959999481836954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,256,0.0029557332396507262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,64,0.0027327999472618104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,32,0.0027200000981489818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,5120,0.24169066747029624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,6144,0.2920629183451334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,7168,0.33169921239217126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,8192,0.3789109230041504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,10240,0.472978146870931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,4096,0.19812266031901044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,2560,0.14494506518046063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,3584,0.17314880688985188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,3072,0.15351467132568358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,1024,0.058347733815511074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,512,0.03811200062433879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,12288,0.5658538818359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,768,0.05276480118433634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,256,0.02162453333536784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,128,0.013378133376439413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,1536,0.08811946709950766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,32,0.010390399893124899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,64,0.010725333293279012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,2048,0.11853226820627849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,7168,0.0835210641225179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,8192,0.09441706339518229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,16384,0.7392234802246094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,10240,0.11892906824747722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,12288,0.13894400596618653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,4096,0.05874133507410685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,5120,0.062242134412129724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,6144,0.07364693482716879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,16384,0.1814304033915202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,3584,0.0460319995880127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,3072,0.044437332948048906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,2560,0.03481173515319824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,768,0.012876799702644348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,1024,0.018436266978581747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,1536,0.02330986658732096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,2048,0.028946133454640706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,256,0.007050666709740956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,512,0.009572266538937887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,128,0.0052159999807675685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,64,0.004583466549714406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,32,0.005110399921735128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,7168,0.06557333469390869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,8192,0.073307736714681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,10240,0.09072746435801188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,12288,0.12430400053660076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,16384,0.15717867215474446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,6144,0.057698134581247965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,5120,0.04903573195139567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,4096,0.03950933218002319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,3584,0.03590826590855916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,3072,0.03118399977684021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,1024,0.012503467003504434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,2048,0.022616533438364665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,2560,0.026946133375167845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,768,0.011352533102035522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,1536,0.018103466431299845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,64,0.004001066585381826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,128,0.0043488000830014546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,256,0.005353599786758423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,512,0.008131200075149536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,32,0.004340266684691111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,8192,0.06414613326390585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,10240,0.07922666867574056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,12288,0.09522773424784342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,16384,0.1436576048533122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,7168,0.05742186705271403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,5120,0.04285226662953694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,6144,0.050429864724477136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,4096,0.03412053187688192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,3072,0.027412267525990804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,2560,0.023364265759785972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,65536,0.7614613215128581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,3584,0.030793599287668866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,2048,0.01987733244895935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,768,0.010517332951227825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,1024,0.011079466342926026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,1536,0.01523413360118866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,512,0.007886933286984761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,256,0.005228800078233084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,128,0.004535466432571411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,64,0.004002133260170618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,32,0.004222933451334635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,65536,0.565890121459961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,12288,0.07240213553110758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,8192,0.04935040076573689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,10240,0.06143573522567749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,16384,0.09302079677581787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,7168,0.05071253379185995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,6144,0.03895573218663533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,5120,0.03332373301188151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,4096,0.02768213351567586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,3584,0.02472213308016459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,2560,0.01887786587079366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,3072,0.021983999013900756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,2048,0.015491200486818948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,1024,0.009180800120035807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,768,0.0081194669008255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,1536,0.012196266651153564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,512,0.006467199822266896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,64,0.003583999971548716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,256,0.004305066665013632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,128,0.0038581334054470064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,32,0.00396373321612676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,65536,0.49356266657511394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,7168,0.045626668135325114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,8192,0.043917866547902425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,10240,0.05453759829203288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,12288,0.06492480039596557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,16384,0.08295146624247232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,6144,0.035316268603007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,4096,0.02446399927139282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,3584,0.022436267137527464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,5120,0.029808000723520918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,3072,0.019988266626993816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,2048,0.013516799608866373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,768,0.007470933099587758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,1536,0.01104960044225057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,2560,0.01925333340962728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,1024,0.008573866883913676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,512,0.005640533566474914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,256,0.004332800209522247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,64,0.0036778666079044344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,128,0.003772799919048945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,32,0.003786666691303253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,65536,0.3990847905476888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,8192,0.03954773346583049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,10240,0.04872533480326335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,12288,0.05792640050252279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,7168,0.04022506475448608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,16384,0.07338559627532959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,4096,0.0221781333287557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,3584,0.02003413240114848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,5120,0.02701440056165059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,6144,0.03157653411229451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,3072,0.017847466468811034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,2560,0.015081600348154704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,768,0.007088000078996022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,1536,0.010365866621335347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,2048,0.01229866643746694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,1024,0.008186666667461396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,512,0.005520000060399374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,256,0.004259199897448222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,128,0.0038090666135152185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,64,0.003505066782236099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,32,0.0036831999818483984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,65536,0.3399903933207194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,7168,0.03592640161514282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,8192,0.034932267665863034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,10240,0.04287573496500651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,12288,0.05106773376464844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,16384,0.0721898635228475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,3072,0.01567359964052836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,5120,0.024156800905863442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,6144,0.028006400664647418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,2560,0.013520000378290811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,3584,0.01808533271153768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,4096,0.019802665710449217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,1024,0.007737599809964498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,65536,0.29179840087890624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,1536,0.009746133287747701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,768,0.006708266834417979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,2048,0.0113237331310908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,512,0.004991999765237173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,256,0.004222933451334635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,128,0.003739733248949051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,64,0.003454933315515518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,32,0.0034805332620938623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,12288,0.0466538667678833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,7168,0.03375146786371867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,8192,0.034509865442911784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,10240,0.04045013189315796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,16384,0.0572106679280599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,6144,0.02977493405342102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,3584,0.013806933164596557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,3072,0.012863999605178833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,5120,0.02035306692123413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,4096,0.015558399756749473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,2560,0.01081706682840983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,65536,0.23840959866841635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,2048,0.009197866916656494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,512,0.0044608001907666525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,1536,0.008004266520341237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,256,0.003853866706291834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,1024,0.006233599781990051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,768,0.005376000205675761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,128,0.003458133339881897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,64,0.0032266666491826378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,32,0.0034901333351929987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,8192,0.024873600403467814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,10240,0.03652799924214681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,12288,0.04184746742248535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,7168,0.022707200050354003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,16384,0.05105706850687662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,6144,0.020093866189320884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,4096,0.01397119959195455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,3584,0.012486400206883748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,65536,0.19378347396850587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,3072,0.011210667093594869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,2560,0.009979733824729919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,5120,0.017171200116475424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,512,0.004053333401679992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,2048,0.008552533388137818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,1536,0.0074421331286430355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,256,0.0035637333989143372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,768,0.00447573314110438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,1024,0.00581226646900177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,128,0.0033429334561030067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,64,0.0033781332274278007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,32,0.003399466723203659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,10240,0.03272533416748047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,12288,0.03742293516794841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,8192,0.02667306661605835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,16384,0.045579731464385986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,7168,0.027291733026504516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,65536,0.1730687936147054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,6144,0.022961066166559855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,5120,0.01636373301347097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,4096,0.01532586713631948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,2560,0.010483200351397198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,3584,0.011688533425331115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,3072,0.010447999835014344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,768,0.00451200008392334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,2048,0.008041599889596303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,1536,0.007067733506361644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,1024,0.00521066685517629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,512,0.003958400090535482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,256,0.0035391998787721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,128,0.0033269333342711128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,64,0.003230933348337809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,32,0.003328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,8192,0.02464639941851298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,10240,0.02879146734873454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,12288,0.03229653239250183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,7168,0.023478400707244874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,16384,0.03980906804402669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,65536,0.15218240420023602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,6144,0.019248000780741372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,5120,0.018551466862360637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,2560,0.009782399733861287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,4096,0.015049599607785544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,3072,0.012418133020401002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,3584,0.012443733215332032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,2048,0.007603199779987335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,1536,0.006554666658242543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,1024,0.004920533299446106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,512,0.003949866692225138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,768,0.0044714664419492085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,256,0.0035125332574049628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,128,0.0033600000043710076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,64,0.0030261332790056865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,32,0.0031082667410373688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,8192,0.023975465695063272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,7168,0.02101226647694906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,10240,0.02453440030415853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,65536,0.1316362698872884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,12288,0.027803732951482134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,16384,0.033505066235860186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,6144,0.018598399559656777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,4096,0.013820800185203552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,5120,0.015758933623631795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,3584,0.012589866916338602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,3072,0.011177600423494975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,1536,0.006111999849478403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,2560,0.008546132842699687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,2048,0.007366399963696797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,1024,0.004613333443800608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,768,0.004182399809360504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,256,0.0034517332911491393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,64,0.0030965333183606463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,512,0.003770666569471359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,128,0.003139200061559677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,32,0.0031776001056035364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,65536,0.10291093190511066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,7168,0.019282132387161255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,8192,0.02135466734568278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,10240,0.02654186685880025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,12288,0.023012266556421915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,16384,0.02759679953257243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,5120,0.014870400230089823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,6144,0.01714026729265849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,3072,0.010136533776919048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,4096,0.012385066350301106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,2560,0.007426133255163829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,3584,0.011313066879908244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,2048,0.00650133341550827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,1024,0.0044170667727788295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,512,0.0037077332536379496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,1536,0.005297066768010458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,768,0.004123733441034953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,256,0.0033183999359607695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,128,0.003049599876006444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,32,0.0030847998956839246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,65536,0.08118933041890462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,64,0.003020799905061722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,8192,0.015703466534614564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,12288,0.02064746618270874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,10240,0.018101332585016887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,6144,0.012425600488980611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,7168,0.013953066865603127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,16384,0.021960532665252684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,5120,0.010841600100199382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,3072,0.008263466755549113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,3584,0.008962133526802063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,4096,0.009525332848230999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,2560,0.007102933526039123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,2048,0.006212266782919565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,1024,0.004466133316357931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,1536,0.0053845331072807315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,768,0.004072533299525579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,65536,0.059432534376780186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,512,0.003638399889071783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,256,0.0032842665910720824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,128,0.002985599885384242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,64,0.0029706666866938275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,32,0.0029525332152843474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,16384,0.02258560061454773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,7168,0.012850133577982583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,12288,0.01843413313229879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,6144,0.011885866522789001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,10240,0.016009599963823954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,8192,0.013142399986584983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,3072,0.007377066711584728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,4096,0.009117866555849712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,5120,0.010597333312034607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,3584,0.008221866687138875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,2560,0.006337066491444905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,2048,0.005637333293755849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,65536,0.049158398310343424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,1536,0.004957866668701172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,1024,0.004242133100827535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,768,0.003969066590070724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,256,0.0032074667513370516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,512,0.003601066768169403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,128,0.0029663999875386557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,32,0.002974933385848999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,64,0.0028917332490285236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,7168,0.011033599575360615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,12288,0.01458346645037333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,10240,0.013192533453305563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,16384,0.01541973352432251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,8192,0.011694932977358501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,6144,0.010340266426404317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,5120,0.009256533781687419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,4096,0.008086400230725606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,3584,0.007849599917729695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,2560,0.0061930666367212926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,3072,0.00719893326361974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,65536,0.037964800993601486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,2048,0.005524266759554545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,1536,0.0048426667849222815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,32,0.002899199972550074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,1024,0.00420906643072764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,768,0.0038261334101359046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,64,0.002807466685771942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,256,0.003142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,512,0.003443199892838796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,128,0.0030389333764712016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,6144,0.006958933174610138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,10240,0.009156266848246258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,16384,0.011496532956759136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,8192,0.0075445334116617845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,7168,0.007417599856853485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,12288,0.010090667009353637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,5120,0.007243733108043671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,4096,0.007169066866238911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,3584,0.007209600011507671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,65536,0.03302826682726542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,3072,0.006839466591676076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,2560,0.006093866626421611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,1536,0.004758400221665701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,2048,0.005452799797058106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,1024,0.004101333270470301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,64,0.0028480000793933867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,768,0.003770666569471359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,512,0.003470933437347412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,256,0.0030730667213598887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,128,0.0029418667157491045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,32,0.0028192001084486645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,16384,0.00876800020535787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,10240,0.007560533285140991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,12288,0.007886933286984761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,8192,0.007089066505432129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,7168,0.006836266815662384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,6144,0.0065653334061304735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,65536,0.023155200481414794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,5120,0.006955733398596446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,4096,0.006589866677920024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,3072,0.006726400057474773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,3584,0.007080533107121785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,2560,0.006042666733264923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,2048,0.005379199981689453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,1024,0.004069333275159201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,768,0.0037087999284267426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,1536,0.004698666433493296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,512,0.0033642667035261786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,256,0.003033600002527237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,32,0.002825599908828735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,128,0.002915200094381968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,64,0.002754133443037669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,16384,0.007159466544787089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,8192,0.006904533505439759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,12288,0.00689279983441035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,10240,0.007014399766921997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,65536,0.013825066884358726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,7168,0.006694399813810985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,6144,0.006485333542029063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,5120,0.006926933427651723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,4096,0.006461866696675618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,3584,0.006915199756622315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,3072,0.006632533172766368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,2560,0.00603413333495458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,768,0.003664000084002813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,2048,0.00535999983549118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,1536,0.004717866579691568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,1024,0.004021333406368891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,64,0.0027872001131375628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,512,0.0033098667860031127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,128,0.0029493334392706556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,256,0.002995199958483378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,32,0.002752000093460083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,16384,0.006855466465155284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,12288,0.006733866532643636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,8192,0.006797866523265838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,10240,0.006911999980608622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,7168,0.006555733581384023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,65536,0.013265066345532737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,6144,0.006450133522351582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,5120,0.006836266815662384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,4096,0.006438399851322174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,3584,0.006764799853165944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,3072,0.006515199939409892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,2560,0.006037333110968272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,2048,0.005353599786758423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,1024,0.004010666658480962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,768,0.003643733263015747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,1536,0.004654933512210846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,512,0.0033386667569478357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,256,0.0030005333324273427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,32,0.0027456000447273255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,128,0.0028736000259717304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,64,0.0027253332237402597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,5120,0.23724586168924966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,6144,0.285256544748942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,7168,0.3271253267923991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,8192,0.3718645413716634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,10240,0.46613334019978836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,3584,0.16967040697733562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,4096,0.193889061609904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,3072,0.15028053919474285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,2560,0.14160854021708172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,2048,0.1156010627746582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,1024,0.06263466676076254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,12288,0.5638271967569988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,768,0.049347201983133956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,1536,0.0892842690149943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,256,0.020709333817164104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,512,0.0328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,128,0.011709866921106975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,32,0.013667200009028116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,64,0.013381333152453104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,7168,0.0835541327794393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,8192,0.09416213035583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,16384,0.7305290857950847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,10240,0.11696000099182129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,12288,0.13931199709574382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,16384,0.18091732660929363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,4096,0.05767893393834432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,5120,0.06162879864374796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,3584,0.045162665843963626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,6144,0.07295146783192953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,3072,0.044887467225392656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,512,0.009318400422732036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,2048,0.027977599700291948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,1024,0.015660799543062844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,768,0.01239466667175293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,1536,0.02318933407465617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,2560,0.038148268063863115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,128,0.004602666695912679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,256,0.006588799754778545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,64,0.004250666499137879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,32,0.00486826648314794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,7168,0.06284586588541666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,8192,0.07069119612375895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,10240,0.08782827059427897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,12288,0.10426026980082195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,16384,0.1357279936472575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,3584,0.034244267145792644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,5120,0.04668906529744466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,4096,0.03832639853159587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,6144,0.05541333357493082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,3072,0.03444799979527791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,1024,0.012071466445922852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,768,0.009859200318654377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,1536,0.01733013391494751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,2048,0.021641600131988525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,2560,0.026065067450205488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,512,0.007812266548474629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,256,0.004937600096066793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,32,0.0043594668308893835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,64,0.003857066730658213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,128,0.0041514667371908825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,8192,0.06012586752573649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,10240,0.07489386399586996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,12288,0.0887775977452596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,16384,0.11481599807739258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,7168,0.05322133302688599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,5120,0.04000106652577718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,6144,0.04722559849421183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,4096,0.034689064820607504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,3072,0.026025599241256712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,2560,0.022487467527389525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,3584,0.029360000292460126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,2048,0.018600533405939736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,1536,0.014683733383814493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,1024,0.010380799571673077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,512,0.006986666719118755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,768,0.008844799796740214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,65536,0.7460277557373047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,64,0.0037280000746250153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,256,0.004738133152325948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,128,0.0039818666875362395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,32,0.004134399940570196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,65536,0.5516661326090495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,8192,0.04851306676864624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,10240,0.06010773181915283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,12288,0.07149439652760824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,16384,0.10355306466420491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,4096,0.026935466130574542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,5120,0.032554666201273605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,6144,0.03815679947535197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,7168,0.050134400526682533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,3584,0.024000000953674317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,1536,0.013345066706339517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,1024,0.009013332923253377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,3072,0.021362133820851645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,2048,0.015107199549674988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,2560,0.01842666665712992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,512,0.006279466549555461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,768,0.007657599945863088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,256,0.004101333270470301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,64,0.0033962666988372804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,128,0.003647999962170919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,32,0.0037802666425704955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,65536,0.48378346761067703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,8192,0.04325013160705567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,10240,0.05382399956385294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,12288,0.07279040018717448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,7168,0.04509333372116089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,16384,0.09136746724446615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,6144,0.03482666810353597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,5120,0.02942720055580139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,4096,0.023986132939656575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,2560,0.01637333333492279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,3072,0.019688532749811808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,3584,0.02135253349939982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,2048,0.013297067085901896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,1536,0.010523733496665955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,1024,0.008171733220418293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,768,0.007049599786599477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,512,0.005467733244101206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,64,0.0033439998825391137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,256,0.003851733356714249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,128,0.003419733295838038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,65536,0.4215242703755696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,32,0.003603200117746989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,7168,0.03351893424987793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,8192,0.043501865863800046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,10240,0.04614293177922567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,12288,0.05467520157496134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,16384,0.07884906927744548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,6144,0.02983466585477193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,5120,0.025417600075403852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,4096,0.021014400323232017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,3584,0.01880853374799093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,2560,0.014113066593805948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,3072,0.016689066092173258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,1536,0.009491200248400371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,2048,0.011662933230400085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,768,0.006488533318042755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,1024,0.00746666689713796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,512,0.004286933441956838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,64,0.003138133386770884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,256,0.0036960000793139136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,128,0.0033610666791598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,65536,0.35250879923502604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,32,0.0035125332574049628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,7168,0.029140265782674153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,8192,0.03718080123265584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,10240,0.04698559840520223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,12288,0.05442986488342285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,16384,0.06751680374145508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,3072,0.013982933759689332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,5120,0.021943465868632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,4096,0.018102399508158364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,6144,0.02609813412030538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,3584,0.01616426706314087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,2560,0.011914666493733723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,2048,0.010190932949384054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,512,0.004117333392302195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,1536,0.00856213370958964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,768,0.005667200187842051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,1024,0.006797866523265838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,65536,0.3043594678243001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,128,0.003365333378314972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,256,0.0036127999424934386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,32,0.0035018667578697203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,64,0.003151999910672506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,12288,0.03883093198140462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,7168,0.024409600098927817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,8192,0.026983465751012164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,10240,0.03312533299128215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,16384,0.066156800587972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,3584,0.013194666306177775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,6144,0.022048000494639078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,4096,0.015135999520619711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,5120,0.019301333030064902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,3072,0.011995733777681986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,65536,0.2533535957336426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,2048,0.009202133615811665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,2560,0.010451199611028035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,1536,0.007630933324495952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,256,0.003685333331425985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,1024,0.006077866752942403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,512,0.004177066683769226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,768,0.004983466863632202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,128,0.0032757334411144257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,32,0.0034389334420363107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,64,0.003138133386770884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,8192,0.024392533302307128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,10240,0.029574400186538695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,7168,0.021826134125391642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,12288,0.03506453434626262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,16384,0.051420799891153966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,65536,0.19464640617370604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,4096,0.013197867075602212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,3072,0.011281067132949829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,6144,0.019581866264343262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,5120,0.016758400201797485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,3584,0.012235732873280843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,2048,0.00865600009759267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,2560,0.011059199770291645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,1536,0.007513600091139476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,1024,0.005456000069777171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,768,0.0046517332394917805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,512,0.004060799876848856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,64,0.0032543999453385672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,256,0.0034954667091369627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,128,0.0033802665770053864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,32,0.0032490665713946023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,7168,0.01990933418273926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,8192,0.02188053329785665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,10240,0.030538666248321533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,12288,0.0350816011428833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,16384,0.04289173285166423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,65536,0.17130667368570965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,3584,0.011545600493748982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,5120,0.016838399569193523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,4096,0.012340266505877178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,6144,0.017734400431315103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,3072,0.010894933342933654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,2048,0.008121599753697712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,2560,0.01030293305714925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,1024,0.005108266572157542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,768,0.00456639975309372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,1536,0.006950399776299794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,256,0.0035957333942254386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,512,0.004012800008058548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,128,0.0031413334111372627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,64,0.003047466774781545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,32,0.0032085334261258446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,7168,0.017767467101415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,8192,0.02015786568323771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,16384,0.03863146702448527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,10240,0.02415999968846639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,65536,0.14679360389709473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,12288,0.028445865710576373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,6144,0.015450666348139444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,4096,0.011220266421635944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,5120,0.013474133610725404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,3072,0.010441600282986959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,3584,0.010381866494814556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,2560,0.008356266220410665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,2048,0.007479466497898102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,1536,0.006390400230884552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,1024,0.004686933259169261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,768,0.0043029333154360454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,512,0.0038346665600935614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,256,0.0033834666013717652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,64,0.0031114667654037476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,128,0.0030858665704727173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,32,0.0030421334008375804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,7168,0.01704960068066915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,8192,0.0191103994846344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,65536,0.1256661335627238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,10240,0.022750933965047203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,12288,0.02579733331998189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,16384,0.03258453408877055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,6144,0.014682666460673014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,3584,0.009735467036565144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,5120,0.013206400473912559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,4096,0.011962667107582092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,2560,0.007837866743405659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,3072,0.009212799866994222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,1536,0.005645866692066193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,1024,0.004561066627502441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,2048,0.006817066669464111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,768,0.004106666644414266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,512,0.003714133302370707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,128,0.00311253344019254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,64,0.002930133293072383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,256,0.0032597333192825317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,32,0.0029290666182835894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,65536,0.10384746392567952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,10240,0.018209065993626913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,7168,0.013911466797192892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,8192,0.014944000045458474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,12288,0.020504534244537354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,16384,0.025143466393152875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,5120,0.010939733187357584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,6144,0.012458667159080505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,4096,0.009469866752624512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,3584,0.008824533224105835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,2048,0.006355200211207073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,3072,0.008088533580303193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,2560,0.00734506646792094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,1536,0.005335466563701629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,1024,0.004494933287302653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,512,0.003642666588226954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,256,0.003269333392381668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,768,0.004109866668780645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,128,0.0030933332939942675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,64,0.002899199972550074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,32,0.0029205332199732465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,65536,0.08021226723988852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,16384,0.0186463991800944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,12288,0.014831999937693277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,8192,0.01125973363717397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,7168,0.012371200323104858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,10240,0.013302399714787801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,6144,0.010418132940928141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,5120,0.010203733046849569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,4096,0.009149866302808125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,3584,0.00819413314263026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,3072,0.007584000130494435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,2048,0.0056320001681645715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,1536,0.004952533543109894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,2560,0.006425599753856659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,65536,0.05720640023549398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,128,0.003048533449570338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,768,0.003918933371702829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,512,0.003602133442958196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,1024,0.004226133227348328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,256,0.00315733328461647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,64,0.0028490667541821797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,32,0.002942933390537898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,6144,0.009547733267148336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,12288,0.013221333424250284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,16384,0.015176533659299215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,8192,0.010689066847165425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,10240,0.01206933359305064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,7168,0.010332799951235453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,5120,0.009406933188438415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,4096,0.008403199911117553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,3584,0.008108800152937572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,3072,0.00717439999183019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,2560,0.00622506688038508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,2048,0.005554133156935374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,1536,0.004849066833655039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,65536,0.05242026646931967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,768,0.003807999938726425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,1024,0.004163199911514918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,64,0.0028629332780838014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,512,0.003437866767247518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,128,0.002867199977238973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,256,0.0030879999200503034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,32,0.002963199963172277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,12288,0.010750933488210043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,16384,0.012162133057912191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,7168,0.009125333031018574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,8192,0.008918399612108868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,10240,0.010404266913731893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,6144,0.009455999732017517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,5120,0.008624000350634257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,4096,0.007525333265463512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,3584,0.0072629332542419435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,2048,0.005526400109132131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,3072,0.006899199883143107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,2560,0.006188799937566122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,65536,0.04079466660817464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,1536,0.004839466512203216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,1024,0.0041407999893029535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,512,0.0034773332377274835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,768,0.0038058665891488397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,64,0.002846933404604594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,256,0.0031317333380381264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,128,0.002889599899450938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,32,0.003102933367093404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,16384,0.010484266281127929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,6144,0.0066549330949783325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,10240,0.007963733375072479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,7168,0.00690773328145345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,8192,0.007134933272997539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,12288,0.00869866708914439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,4096,0.006668800115585327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,65536,0.029691733916600543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,5120,0.007144533097743988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,3584,0.007096533477306366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,3072,0.006776533524195353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,2560,0.006075733403364817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,2048,0.0053951998551686605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,1536,0.004740266501903534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,768,0.003692800054947535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,1024,0.004070399949947992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,512,0.0033781332274278007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,32,0.0028437333802382152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,256,0.0030229332546393077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,128,0.002922666569550832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,64,0.0027594665686289472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,12288,0.006822399795055389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,16384,0.007419733206431071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,10240,0.006982400019963582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,8192,0.006854400038719177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,65536,0.017732266585032144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,6144,0.006626133124033611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,7168,0.006791466474533081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,3584,0.007063466807206471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,5120,0.006975999971230824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,4096,0.00660693347454071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,2560,0.006066133578618368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,1024,0.004043733328580856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,3072,0.0067210664351781205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,2048,0.005387733379999796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,1536,0.004709333181381226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,768,0.0036874666810035707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,512,0.0033674667278925574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,128,0.0029696000119050344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,256,0.003124266614516576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,32,0.0027445333699385325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,64,0.0027583998938401537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,16384,0.006764799853165944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,12288,0.006640000144640605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,10240,0.006883200009663899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,8192,0.006896000107129414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,65536,0.013041067123413085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,7168,0.006613333523273468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,6144,0.006534400085608165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,3584,0.006982400019963582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,4096,0.006514133512973785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,5120,0.006850133339564006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,3072,0.006698666512966156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,2560,0.006027733286221823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,2048,0.005350400010744731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,1536,0.004694400231043497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,1024,0.004020266731580098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,256,0.003031466652949651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,768,0.003676799933115641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,512,0.003316266586383184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,128,0.002930133293072383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,16384,0.0067104001839955645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,32,0.002755200117826462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,64,0.002755200117826462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,10240,0.006753066678841908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,12288,0.006555733581384023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,7168,0.006542933483918508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,65536,0.011291733384132386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,8192,0.0066431999206542965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,6144,0.006422399977842967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,5120,0.006786133348941803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,3584,0.006712533533573151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,4096,0.006251733501752217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,3072,0.006390400230884552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,2560,0.006045866509278615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,2048,0.005393066505591074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,1536,0.0046858668327331545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,1024,0.004023466755946477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,256,0.003099733342727025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,768,0.003669333209594091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,512,0.0033621333539485933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,128,0.002922666569550832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,64,0.002780800064404805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,32,0.0026528000831604003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,5120,0.23372373580932618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,6144,0.28023999532063804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,7168,0.32161280314127605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,8192,0.3681973457336426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,4096,0.18966293334960938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,10240,0.4579135894775391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,3072,0.14738240242004394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,3584,0.16633599599202473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,2560,0.1387466589609782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,12288,0.5512725194295247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,768,0.047270401318868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,1024,0.057905066013336184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,64,0.008039466540018718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,512,0.031096533934275312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,256,0.019896533091862997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,128,0.01046399970849355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,1536,0.08645866711934408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,32,0.0072512000799179075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,2048,0.10172266960144043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,7168,0.0814197301864624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,8192,0.09198079903920492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,10240,0.11387093067169189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,12288,0.1361013412475586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,16384,0.7313653310139974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,16384,0.17941120465596516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,5120,0.059955199559529625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,6144,0.0713375965754191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,4096,0.04915519952774048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,3584,0.04634453455607097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,3072,0.044061867396036784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,2048,0.027360000212987262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,2560,0.037251198291778566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,768,0.011873066425323486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,1536,0.021691733598709108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,1024,0.01525759994983673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,256,0.006365866462389629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,512,0.009090133508046468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,128,0.004263466596603394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,64,0.003941333293914795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,32,0.004267733295758565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,8192,0.07181546688079835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,7168,0.06332693497339884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,10240,0.08695573012034098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,12288,0.10509653091430664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,16384,0.14975892702738444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,4096,0.0377461314201355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,5120,0.047133866945902506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,6144,0.05629440148671468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,3584,0.0336575984954834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,3072,0.03321173389752706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,2560,0.02558506727218628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,1024,0.011934933066368104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,2048,0.021219199895858763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,1536,0.017011199394861856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,512,0.007751466830571492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,768,0.00983786682287852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,256,0.004524800181388855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,32,0.0038805333276589714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,128,0.0038474666575590765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,64,0.0035743998984495797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,8192,0.05915946563084921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,10240,0.07370773156483969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,12288,0.08761386871337891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,16384,0.13243199984232584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,5120,0.03946453332901001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,7168,0.05254293282826742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,6144,0.04675413370132446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,4096,0.03210666577021281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,3584,0.02917226751645406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,3072,0.025906133651733398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,2560,0.022223999102910362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,1536,0.014696533481280008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,2048,0.018453333775202432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,512,0.007272533575693766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,768,0.00897279977798462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,1024,0.010580266515413921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,128,0.003866666555404663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,65536,0.7313653310139974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,256,0.0043381333351135256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,64,0.0035434665779272715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,32,0.003642666588226954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,8192,0.0480128010114034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,10240,0.05907946825027466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,12288,0.07047359943389893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,65536,0.5400394439697266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,16384,0.11638826529184979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,4096,0.030001066128412884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,5120,0.032085333267847696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,7168,0.042718935012817386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,6144,0.03785706758499145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,3584,0.026950399080912273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,1536,0.011610666910807293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,1024,0.008814932902654012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,2048,0.01474346617857615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,3072,0.02376213272412618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,2560,0.01803413430849711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,512,0.006066133578618368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,768,0.00745066652695338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,256,0.004218666752179464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,128,0.003554133325815201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,64,0.0034005333979924522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,65536,0.47189760208129883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,32,0.003732266773780187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,8192,0.04935786724090576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,7168,0.03784639835357666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,10240,0.052831999460856115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,12288,0.06219520171483358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,16384,0.08983039855957031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,6144,0.03380053440729777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,5120,0.028549333413441975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,3584,0.021248000860214233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,4096,0.02381226619084676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,2560,0.018360533316930137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,2048,0.013343999783198038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,3072,0.01946773330370585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,1536,0.010651733477910359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,1024,0.008339200417200725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,768,0.007091199855009715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,65536,0.3837717374165853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,512,0.0048885335524876915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,256,0.003912533322970072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,32,0.003491200009981791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,128,0.003435733417669932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,64,0.003421866645415624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,10240,0.045553068319956466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,8192,0.037109335263570145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,7168,0.033352533976236984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,12288,0.05463146766026815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,16384,0.08073813120524088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,6144,0.029632000128428142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,5120,0.025191466013590496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,4096,0.020967467625935873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,3584,0.018744534254074095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,3072,0.01662613352139791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,2560,0.013920000195503235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,2048,0.01204159955183665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,1536,0.009708799918492635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,512,0.004339199761549632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,1024,0.007705600063006084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,768,0.006724266707897187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,256,0.003833599885304769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,64,0.0034400001168251038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,128,0.0033439998825391137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,65536,0.3332255999247233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,32,0.003425066669782003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,12288,0.05418239831924439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,7168,0.028823467095692952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,8192,0.0369322657585144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,10240,0.03947413365046183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,16384,0.06820800304412841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,6144,0.025796266396840413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,4096,0.01800640026728312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,5120,0.02194026708602905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,3072,0.014335999886194864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,3584,0.016301866372426352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,2560,0.012095999717712403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,65536,0.2805962562561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,1024,0.006894933183987935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,2048,0.010379733641942342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,1536,0.008628267049789428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,768,0.005547733108202616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,512,0.004156800111134847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,256,0.0036650667587916053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,128,0.0032810665667057036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,64,0.003050666550795237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,32,0.0032074667513370516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,12288,0.042881067593892416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,8192,0.03113173246383667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,10240,0.03704640070597331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,7168,0.02856106758117676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,16384,0.0531274676322937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,3072,0.011921067039171855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,3584,0.013147733608881631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,6144,0.02577706575393677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,4096,0.01493119994799296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,5120,0.018154666821161906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,65536,0.23405119578043618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,2560,0.01051093339920044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,2048,0.00909546713034312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,1536,0.007642666498819987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,1024,0.006209066510200501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,256,0.0034613333642482757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,768,0.0045962666471799215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,512,0.0038165333370367683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,128,0.0031615999837716425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,64,0.003089066594839096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,32,0.0033600000043710076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,8192,0.028300799926122028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,12288,0.03873066504796346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,7168,0.022929066419601442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,10240,0.03349866469701131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,16384,0.04793813228607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,6144,0.02039253314336141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,4096,0.01320746640364329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,65536,0.1903253396352132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,3584,0.012114133437474568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,5120,0.016317866245905557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,3072,0.010863999525705974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,2560,0.00973653296629588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,2048,0.008332799871762593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,256,0.0034517332911491393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,1536,0.007897600034872691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,768,0.004222933451334635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,512,0.00377813329299291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,1024,0.005319466690222422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,64,0.00311253344019254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,128,0.0031583999594052637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,32,0.0031509332358837126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,10240,0.02967573404312134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,8192,0.0252895991007487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,12288,0.03437439997990926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,16384,0.04556586742401123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,7168,0.0212501327196757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,65536,0.17367679278055828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,5120,0.014477866888046264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,2560,0.009876267115275065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,4096,0.012025599678357441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,6144,0.01729066570599874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,3584,0.011085866888364156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,3072,0.009920000036557516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,1536,0.006780800223350525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,2048,0.007815466821193695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,768,0.004308266441027323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,1024,0.004926933348178864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,256,0.0033429334561030067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,512,0.003774933268626531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,128,0.003253333270549774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,64,0.0029674666623274487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,32,0.0031285333136717476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,7168,0.01939199964205424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,12288,0.029296000798543293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,8192,0.021729065974553426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,10240,0.025788799921671553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,65536,0.1462165355682373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,16384,0.03635093371073405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,6144,0.01739733417828878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,3584,0.01002346674601237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,4096,0.012575999895731608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,5120,0.012826666235923767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,3072,0.010028800368309021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,2560,0.008342400193214417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,2048,0.00727893312772115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,1536,0.006138666470845541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,1024,0.00452693353096644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,768,0.0042133331298828125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,512,0.0037664001186688742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,128,0.0032266666491826378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,64,0.0030037333567937215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,256,0.0034293333689371743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,32,0.003018666555484136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,65536,0.12384213606516521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,7168,0.016200533509254454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,10240,0.021874133745829263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,12288,0.024920533100763954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,8192,0.018172800540924072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,16384,0.030435200532277423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,6144,0.014990933736165366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,4096,0.01058666706085205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,5120,0.012852266430854797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,3584,0.00974079966545105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,2560,0.007985066870848339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,3072,0.008664533495903015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,1536,0.005719466507434845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,1024,0.0045056000351905824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,2048,0.007546666761239369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,768,0.0040618665516376495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,512,0.0036661334335803984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,128,0.0030965333183606463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,256,0.0032757334411144257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,64,0.0028789333999156954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,32,0.0031040000418821974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,65536,0.10232426325480144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,12288,0.020307199160257975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,8192,0.015097600221633912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,10240,0.017850667238235474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,16384,0.0250709335009257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,7168,0.01378986636797587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,6144,0.01234879990418752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,5120,0.010913067062695821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,4096,0.009455999732017517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,3584,0.00881173312664032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,3072,0.008125866452852886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,1536,0.005142400165398916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,2560,0.007163733243942261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,2048,0.0061034664511680605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,1024,0.004462933540344239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,256,0.0032927999893824257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,512,0.0036960000793139136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,65536,0.08039893309275309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,768,0.004138666639725367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,128,0.0030080000559488933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,64,0.0028586665789286296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,32,0.002962133288383484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,12288,0.014652799566586813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,8192,0.01316480040550232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,10240,0.014003200332323709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,16384,0.01842666665712992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,7168,0.012172800302505494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,6144,0.011332266529401143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,5120,0.010198400417963664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,1536,0.004863999783992767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,3584,0.008525866270065307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,4096,0.009412266810735067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,3072,0.007715199887752533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,2048,0.005532800157864889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,2560,0.006253866851329804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,1024,0.004133333265781402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,65536,0.060488533973693845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,768,0.003828266759713491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,128,0.0029557332396507262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,512,0.0035146666069825493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,256,0.0031445334355036415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,32,0.0028768000503381092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,64,0.002739199995994568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,7168,0.010965333382288615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,12288,0.01447466711203257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,10240,0.013170133034388224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,8192,0.01176533301671346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,16384,0.01731946667035421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,6144,0.010286933183670044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,5120,0.009439999858538311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,4096,0.008293333152929943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,3072,0.007114666700363159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,1536,0.00483840008576711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,3584,0.007705600063006084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,2560,0.006182399888833364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,2048,0.005494399865468343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,65536,0.04866986672083537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,512,0.0035114665826161706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,1024,0.004155733436346054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,768,0.0038293334345022834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,256,0.0031968000034491217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,64,0.0027978666126728057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,128,0.0029493334392706556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,32,0.0029088000456492106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,16384,0.013581867019335428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,6144,0.008091733356316884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,12288,0.011789866288503011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,8192,0.009779199957847595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,10240,0.010847999652226766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,7168,0.009060266613960265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,4096,0.00682773341735204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,5120,0.007541333138942718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,3584,0.00721919983625412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,65536,0.037453866004943846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,3072,0.006821333368619282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,2560,0.006251733501752217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,2048,0.005520000060399374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,1536,0.004811733464399974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,1024,0.004186666508515676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,768,0.00378560001651446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,512,0.00347626656293869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,32,0.0028597332537174227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,256,0.0030773334205150605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,128,0.0029056000212828318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,64,0.0028031999866167706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,10240,0.007815466821193695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,16384,0.010121599833170573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,12288,0.00853546659151713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,8192,0.007084799806276958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,7168,0.006938666601975759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,6144,0.00660693347454071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,5120,0.0070783997575442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,65536,0.02368639906247457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,4096,0.006678399940331777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,3584,0.007123200098673503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,3072,0.006746666630109151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,2560,0.006073600053787232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,2048,0.005389866729577383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,1536,0.004728533327579498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,1024,0.0040618665516376495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,512,0.0034314667185147605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,768,0.003689600030581156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,32,0.002792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,256,0.002985599885384242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,128,0.002959999938805898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,64,0.0027647999425729113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,12288,0.006809600194295247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,16384,0.00739519993464152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,10240,0.006934399902820587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,8192,0.00693333347638448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,65536,0.014913066228230795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,7168,0.0067445332805315655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,6144,0.00658133327960968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,3584,0.007019733389218648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,5120,0.006941866874694824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,4096,0.006512000163396199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,2560,0.006050133208433787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,3072,0.006705066561698914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,1024,0.004062933226426443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,2048,0.005363200108210246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,1536,0.004714666803677877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,768,0.003684266656637192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,512,0.003402666747570038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,64,0.002808533360560735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,256,0.0030645333230495454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,128,0.0028917332490285236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,32,0.0028480000793933867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,12288,0.006716800232728322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,16384,0.006712533533573151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,10240,0.006840533514817555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,8192,0.006759466727574666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,7168,0.006638933221499126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,65536,0.012729600071907043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,3584,0.006903466582298279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,6144,0.006542933483918508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,5120,0.006903466582298279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,4096,0.00644053320089976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,3072,0.006603733201821645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,1536,0.004665599763393402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,2560,0.005998933315277099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,2048,0.005358933409055074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,768,0.0036799999574820197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,512,0.0033290666838486993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,1024,0.00402453343073527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,128,0.00290133332212766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,256,0.002994133283694585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,64,0.002737066646416982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,32,0.002738133321205775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,10240,0.006809600194295247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,16384,0.006664533416430156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,12288,0.00658133327960968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,8192,0.006625066697597504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,65536,0.011260799566904704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,7168,0.006586666901906331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,5120,0.006764799853165944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,6144,0.006420266628265381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,4096,0.006357333560784657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,3072,0.006377600133419037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,3584,0.00668693333864212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,2560,0.006011733412742614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,1536,0.004711466530958811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,2048,0.005358933409055074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,1024,0.004019200056791306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,768,0.003664000084002813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,64,0.002776533365249634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,128,0.002880000074704488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,512,0.0033258666594823206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,256,0.003047466774781545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,32,0.002656000107526779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,5120,0.23953065872192383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,6144,0.28647467295328777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,7168,0.333516788482666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,8192,0.378987725575765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,10240,0.47441600163777664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,4096,0.19413973490397135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,3584,0.172542937596639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,2560,0.1451530615488688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,3072,0.15195412635803224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,768,0.04357226689656575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,1024,0.06224639813105265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,512,0.03517013390858968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,1536,0.08626240094502767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,12288,0.5717461268107097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,2048,0.11533652941385905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,256,0.018602667252222697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,128,0.01111466685930888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,64,0.008667733271916707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,32,0.007319466769695282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,7168,0.08142293294270833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,8192,0.0920576016108195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,10240,0.1142357349395752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,12288,0.13639893531799316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,16384,0.7372053146362305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,16384,0.17928320566813152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,5120,0.05959039926528931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,6144,0.07148266633351644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,3584,0.04955093463261922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,4096,0.04885653257369995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,3072,0.04374293486277263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,1024,0.015110400319099427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,768,0.011766399939854939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,2560,0.03702400128046672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,1536,0.021466666460037233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,2048,0.027320533990859985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,256,0.006287999947865804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,128,0.004147200038035711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,512,0.008980266253153483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,64,0.00383146678407987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,32,0.00410453329483668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,8192,0.0722325325012207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,7168,0.06639999945958455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,10240,0.09311253229777018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,12288,0.11150720119476318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,16384,0.14810667037963868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,5120,0.04837760130564372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,6144,0.056066131591796874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,4096,0.040090668201446536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,3584,0.03578986724217732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,3072,0.03350720008214315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,2048,0.021687465906143188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,1024,0.012614400188128153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,2560,0.02661866744359334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,1536,0.017373865842819212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,512,0.0077909335494041445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,768,0.010099200407663982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,256,0.004763733347256978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,128,0.0038143999874591826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,32,0.0037962667644023894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,64,0.003483733286460241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,10240,0.0741749366124471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,8192,0.05868479808171591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,12288,0.08819200197855631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,16384,0.1299349308013916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,7168,0.05287359952926636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,6144,0.046062934398651126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,5120,0.039399464925130204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,4096,0.032713599999745685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,3584,0.02886506716410319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,3072,0.025283199548721314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,2560,0.022046933571497597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,1536,0.014338133732477823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,2048,0.018513067563374837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,1024,0.010378666718800863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,512,0.0071381335457166035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,768,0.00890239973862966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,65536,0.7509429295857747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,256,0.0042463997999827065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,64,0.0034517332911491393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,128,0.0037813333173592886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,32,0.0035648000737031303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,8192,0.04759893417358398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,12288,0.06990933418273926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,65536,0.5381610870361329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,10240,0.058746667702992764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,16384,0.09963093598683676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,4096,0.02625173330307007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,6144,0.03766186634699504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,5120,0.036298668384552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,7168,0.04226453304290771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,3584,0.02342933416366577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,2560,0.017816533644994102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,1536,0.011498666803042094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,2048,0.016634666919708253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,1024,0.00876586635907491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,3072,0.020702934265136717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,768,0.007451733450094859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,512,0.005938133100668589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,65536,0.4695007960001628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,256,0.004103466620047887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,128,0.003533866753180822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,32,0.003623466690381368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,64,0.0034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,8192,0.04244693517684937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,7168,0.037783467769622804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,10240,0.0520032008488973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,12288,0.061963733037312826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,16384,0.10128533045450847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,6144,0.03364373445510864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,5120,0.02847786744435628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,4096,0.023447465896606446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,3584,0.02103040019671122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,3072,0.020848000049591066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,2048,0.013355732957522074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,2560,0.016089600324630738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,1536,0.010634666681289673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,768,0.00701333334048589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,1024,0.00820906658967336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,64,0.003446399917205175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,256,0.0037941334148248037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,512,0.004830933113892873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,32,0.003470933437347412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,128,0.003398400048414866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,65536,0.39902505874633787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,7168,0.037931732336680096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,8192,0.03693653345108032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,10240,0.045579731464385986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,12288,0.05402559836705526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,16384,0.07983360290527344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,5120,0.02502826650937398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,6144,0.029665066798528032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,4096,0.021041067441304524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,3072,0.016369066635767617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,3584,0.018689066171646118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,1024,0.0074986666440963745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,1536,0.00983679989973704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,2048,0.012001066406567892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,2560,0.014007467031478881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,768,0.006626133124033611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,512,0.004267733295758565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,256,0.0037503999968369803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,128,0.003403733422358831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,64,0.003171200056870779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,65536,0.3378719965616862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,32,0.0032330666979153953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,7168,0.028326400121053058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,8192,0.036474665006001786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,12288,0.05328853527704874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,10240,0.03887573480606079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,16384,0.06730986436208089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,3072,0.0139957328637441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,4096,0.017798399925231932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,3584,0.015970133741696677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,6144,0.02526400089263916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,5120,0.02145599921544393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,65536,0.27898985544840493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,2560,0.011982933680216471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,1536,0.008610133330027263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,2048,0.010246400038401287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,1024,0.00676800012588501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,512,0.004301866888999939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,768,0.005857066810131073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,256,0.003607466568549474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,64,0.0031744000812371576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,128,0.003310933212439219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,32,0.0031093334158261614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,8192,0.030717867612838744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,10240,0.03634346723556518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,7168,0.027908267577489217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,12288,0.0422165314356486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,16384,0.05229653517405192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,65536,0.2334346612294515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,3072,0.011943466464678447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,3584,0.013304533561070761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,6144,0.025206400950749712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,5120,0.018286933501561485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,4096,0.015082666277885437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,2560,0.011741866668065388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,2048,0.009134933352470398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,1536,0.007699200014273326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,256,0.003420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,768,0.004403199752171834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,1024,0.006129066646099091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,512,0.0038250667353471124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,128,0.0030794667700926462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,32,0.0033621333539485933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,64,0.0029728000362714132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,8192,0.02781013250350952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,7168,0.021297067403793335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,10240,0.032874667644500734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,12288,0.038200533390045165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,16384,0.0490506649017334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,65536,0.1885109265645345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,6144,0.019339734315872194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,4096,0.013204266627629599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,2560,0.01076693336168925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,5120,0.016229333480199178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,3584,0.012153599659601848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,3072,0.010892800490061442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,1024,0.005338666836420695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,2048,0.008620799581209818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,1536,0.007319466769695282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,768,0.004144000013669332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,256,0.0033600000043710076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,512,0.0037994667887687682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,128,0.0031776001056035364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,64,0.003058133274316788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,32,0.0030026666820049284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,8192,0.02421226700146993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,10240,0.028381866216659547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,7168,0.02218559980392456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,12288,0.03358613252639771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,16384,0.044147201379140216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,65536,0.174181334177653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,5120,0.017155200242996216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,4096,0.015159466862678527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,3584,0.01092906693617503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,6144,0.01984000007311503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,3072,0.010179199775060018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,2048,0.007919999957084655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,2560,0.00976639986038208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,1536,0.0068810666600863145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,256,0.00345920001467069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,1024,0.004941866795221964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,768,0.004162133236726125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,512,0.003755733370780945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,128,0.0030858665704727173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,64,0.002906666696071625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,32,0.003020799905061722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,7168,0.019400533040364584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,8192,0.021317332983016968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,10240,0.025234133005142212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,12288,0.028749867280324297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,16384,0.03592746655146281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,65536,0.14534506797790528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,6144,0.017028266191482545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,5120,0.01300373375415802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,3072,0.009236266215642292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,4096,0.010972799857457478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,2560,0.008929066856702169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,3584,0.010103467106819152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,2048,0.00726506660381953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,1536,0.006312533219655354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,1024,0.004496000210444133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,768,0.004131199916203817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,512,0.003706666578849157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,256,0.0033749334514141084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,128,0.0032032000521818793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,32,0.0030602666238943735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,64,0.0029098667204380036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,7168,0.016484266519546507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,10240,0.021282132466634116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,65536,0.12310079733530681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,12288,0.02434026598930359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,8192,0.017986132701237997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,16384,0.029977599779764812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,3072,0.009041066964467366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,6144,0.015016532937685647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,5120,0.012874666849772134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,4096,0.011055999994277954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,3584,0.010821333527565003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,2560,0.00773119976123174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,2048,0.006658133367697399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,1536,0.005444266895453135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,1024,0.004335999985535939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,768,0.003962666789690653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,256,0.0032416000962257386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,512,0.003634133438269297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,128,0.003101866692304611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,64,0.0029088000456492106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,32,0.002961066613594691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,65536,0.10138239860534667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,10240,0.01723946730295817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,7168,0.013274666666984559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,8192,0.014222932855288186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,12288,0.01954666574796041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,16384,0.024515199661254882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,6144,0.01213653286298116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,4096,0.010020266969998677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,5120,0.01109333336353302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,3584,0.008640000224113464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,2560,0.007237333556016285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,3072,0.009846400221188862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,1536,0.005008000135421753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,2048,0.006239999830722809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,1024,0.0042912001411120095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,768,0.003955200066169103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,512,0.0035946667194366455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,128,0.003091199944416682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,256,0.003289599965016047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,64,0.0029567999144395193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,32,0.003010133405526479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,65536,0.07947946389516194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,12288,0.014359466234842935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,7168,0.010686933000882467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,8192,0.011329066753387452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,10240,0.012829867005348206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,6144,0.01188693344593048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,16384,0.017895466089248656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,4096,0.009155199925104777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,5120,0.010526933272679647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,3584,0.008543999989827473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,3072,0.007772799829641979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,2560,0.00689279983441035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,2048,0.005673600236574808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,1536,0.005163733164469401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,65536,0.059087999661763514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,1024,0.0043712000052134195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,768,0.0039061332742373147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,64,0.0029109333952267963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,128,0.002976000060637792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,512,0.003583999971548716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,256,0.0031189332405726117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,32,0.0028597332537174227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,12288,0.012370133399963379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,16384,0.01463573376337687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,10240,0.013676800330479941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,8192,0.01200213332970937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,7168,0.010900266965230306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,6144,0.010540800293286641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,5120,0.00907306671142578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,4096,0.008053333560625712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,3584,0.00774186650911967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,1536,0.00486826648314794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,65536,0.04784746567408244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,3072,0.007152000069618225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,2560,0.006251733501752217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,2048,0.005520000060399374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,1024,0.004156800111134847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,512,0.00346666673819224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,768,0.0038143999874591826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,256,0.0032405334214369455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,32,0.002880000074704488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,128,0.0030517332255840302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,64,0.0028543998797734577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,16384,0.011805867155392964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,10240,0.011058132847150166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,12288,0.01053013304869334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,8192,0.009780266880989074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,7168,0.008948266506195068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,6144,0.009777067104975383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,4096,0.0078005333741505934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,5120,0.008624000350634257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,65536,0.037028264999389646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,3584,0.007201066613197327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,3072,0.006857599814732869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,1536,0.004739200075467428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,2560,0.006151466568311056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,2048,0.005409066875775656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,1024,0.004098133246103922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,768,0.0037621334195137024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,512,0.003416533271471659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,128,0.0029909332593282064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,256,0.0030389333764712016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,64,0.0028586665789286296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,32,0.0028351999819278715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,10240,0.008631466825803121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,8192,0.007341866691907247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,16384,0.011064533392588298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,12288,0.009294933080673218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,5120,0.007108266651630402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,7168,0.007032533486684163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,6144,0.006658133367697399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,65536,0.022806400060653688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,4096,0.0066890666882197065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,3584,0.00699839989344279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,3072,0.006669866542021434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,2560,0.006044800082842508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,2048,0.0053951998551686605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,1536,0.0047423998514811196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,1024,0.004051200052102407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,128,0.002946133414904277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,768,0.0036703998843828833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,512,0.0033600000043710076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,256,0.0030794667700926462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,16384,0.007406933108965556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,64,0.0027978666126728057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,32,0.00276053324341774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,12288,0.007354666789372762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,8192,0.006857599814732869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,10240,0.00698773314555486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,65536,0.014660267035166421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,7168,0.006743466854095459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,6144,0.00652266691128413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,5120,0.006904533505439759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,4096,0.006478933493296306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,3584,0.006980266670385997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,2048,0.005340800185998281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,3072,0.006623999774456024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,2560,0.00602346658706665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,1536,0.004717866579691568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,256,0.0031189332405726117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,768,0.0036906667053699495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,1024,0.004025600105524063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,512,0.003382399926582972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,64,0.0027647999425729113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,128,0.0028778667251269023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,32,0.002809600035349528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,8192,0.006715733309586842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,16384,0.006693333387374878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,10240,0.006828799843788147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,7168,0.00652266691128413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,12288,0.006638933221499126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,65536,0.012802132964134216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,5120,0.006862933437029521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,6144,0.006466133395830791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,4096,0.006426666676998138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,3072,0.006526933113733928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,3584,0.006878933310508728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,2560,0.006027733286221823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,1536,0.004671999812126159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,2048,0.005356800059477488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,768,0.003659733384847641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,256,0.002974933385848999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,512,0.003339733431736628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,1024,0.004040533304214477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,64,0.002773333340883255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,32,0.002752000093460083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,128,0.002842666705449422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,8192,0.006631466746330261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,16384,0.006951466699441274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,12288,0.006656000018119812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,7168,0.006534400085608165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,10240,0.006807466844717662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,65536,0.010974933703740437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,6144,0.006392533580462138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,3072,0.00652159998814265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,5120,0.006761600077152252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,3584,0.00689386675755183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,4096,0.00639573335647583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,2560,0.006026666859785716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,2048,0.005365333457787832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,1536,0.004668800036112467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,1024,0.004030933231115341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,768,0.003659733384847641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,512,0.003349333256483078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,256,0.0030421334008375804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,128,0.002916266769170761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,64,0.0026591998835404714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,32,0.0027306665976842242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,5120,0.2384042739868164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,6144,0.28559786478678384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,7168,0.3323552131652832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,8192,0.3757002512613932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,10240,0.4731018702189128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,4096,0.1913365364074707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,3584,0.170579195022583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,3072,0.1500970681508382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,2560,0.1446794668833415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,12288,0.5681397120157878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,1024,0.06066986719767252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,768,0.04793279965718587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,256,0.019568000237147012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,128,0.01034453312555949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,1536,0.08525760173797607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,512,0.03021120031674703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,32,0.006313600142796834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,64,0.007965866724650066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,2048,0.11378346284230549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,7168,0.08188479741414388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,16384,0.7391509373982748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,8192,0.09456106821695963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,10240,0.11738026936848958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,12288,0.13806506792704265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,4096,0.04825280110041301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,3584,0.042770131429036455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,5120,0.06928533713022868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,6144,0.07146026293436686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,16384,0.19560532569885253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,3072,0.03935893376668294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,2048,0.029892265796661377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,1536,0.020909865697224937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,1024,0.014745600024859109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,768,0.0132341335217158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,512,0.008776533603668212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,2560,0.03199573357899983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,256,0.006049066781997681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,128,0.003939199944337209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,64,0.003684266656637192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,32,0.0038751999537150065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,7168,0.06132800181706747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,8192,0.06909120082855225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,10240,0.08549866676330567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,12288,0.1023360013961792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,16384,0.13373866081237792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,4096,0.0370250662167867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,5120,0.04522666533788045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,3584,0.033129600683848064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,6144,0.05391039848327637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,1024,0.011389866471290588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,2560,0.024690133333206177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,1536,0.0161461333433787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,3072,0.032798933982849124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,2048,0.020360533396402994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,256,0.0041354666153589886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,512,0.007260799904664357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,768,0.009314133723576864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,128,0.0036618667344252265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,64,0.0033941333492596946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,32,0.0035146666069825493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,8192,0.0580618659655253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,10240,0.07149973710378012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,12288,0.08591573238372803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,16384,0.11196800072987874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,7168,0.05140906572341919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,6144,0.04544853369394938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,5120,0.03811519940694173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,3584,0.027913600206375122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,4096,0.03160640001296997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,3072,0.02462293306986491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,2560,0.021167999505996703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,1536,0.013731200496355692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,2048,0.01755839983622233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,1024,0.009938133756319682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,256,0.004006399959325791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,768,0.008363733688990276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,512,0.006704000135262807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,64,0.003401600072781245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,128,0.003565866748491923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,65536,0.71484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,32,0.00346666673819224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,8192,0.04734186728795369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,65536,0.5556693394978841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,10240,0.06774400075276693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,12288,0.06954346497853597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,16384,0.09052373568216959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,7168,0.04219520092010498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,3584,0.023116799195607503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,4096,0.025983999172846477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,5120,0.036064000924428304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,6144,0.042813865343729655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,2560,0.01765226721763611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,1024,0.009756799538930256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,1536,0.011494400103886922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,2048,0.014377599954605103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,3072,0.020503467321395873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,768,0.00738560010989507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,512,0.005656533439954122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,256,0.003832533210515976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,128,0.0034656000634034475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,65536,0.4601258595784505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,32,0.0034111998975276947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,64,0.0033482665816942847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,8192,0.04345279932022095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,7168,0.03787519931793213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,10240,0.059779198964436855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,12288,0.06164480050404867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,16384,0.086244265238444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,4096,0.023398399353027344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,5120,0.02871893246968587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,6144,0.033758934338887533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,3584,0.02141653299331665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,1536,0.01045973300933838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,2048,0.01356160044670105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,1024,0.00842133363087972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,3072,0.01900586684544881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,2560,0.01565439999103546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,768,0.007114666700363159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,256,0.003705599904060364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,512,0.00452693353096644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,128,0.003389866650104523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,64,0.0033130665620168054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,32,0.0033002667129039764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,65536,0.37231785456339517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,8192,0.03722026745478312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,7168,0.038680533568064376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,10240,0.05
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,12288,0.059102932612101235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,16384,0.07469653288523356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,6144,0.029395200808842975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,4096,0.020510933796564736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,5120,0.024871466557184856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,3584,0.018270933628082277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,2048,0.012098133563995361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,2560,0.014694399634997048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,3072,0.01621119976043701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,1536,0.009682133793830872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,1024,0.007625600198904674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,768,0.006623999774456024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,512,0.004154666761557261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,256,0.0035957333942254386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,32,0.003236266722281774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,128,0.0032810665667057036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,64,0.0031114667654037476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,65536,0.337499745686849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,12288,0.05007253487904867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,10240,0.03819093306859334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,7168,0.03205546736717224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,8192,0.031429332494735715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,16384,0.0629792014757792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,6144,0.025067732731501265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,3072,0.013965866963068643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,4096,0.017821866273880004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,3584,0.015850667158762613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,5120,0.021155200401941934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,2560,0.01200320025285085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,2048,0.010295466581980387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,1536,0.008603733777999879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,512,0.0038624001046021783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,65536,0.27528320948282875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,1024,0.0068458666404088335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,768,0.0055402666330337524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,64,0.0030805334448814393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,256,0.003499733408292135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,128,0.003155199935038885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,32,0.003081600119670232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,12288,0.04139519929885864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,8192,0.02945493261019389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,7168,0.026739199956258137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,10240,0.035293865203857425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,16384,0.055061332384745275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,3584,0.013196800152460733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,6144,0.021000534296035767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,4096,0.014788267016410828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,3072,0.01195733348528544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,5120,0.018158932526906334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,65536,0.2308351993560791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,2048,0.009524266918500264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,2560,0.010297600428263347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,1536,0.007515733440717061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,512,0.003738666574160258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,768,0.0041802664597829185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,1024,0.0061365331212679545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,256,0.0033749334514141084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,32,0.0030229332546393077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,64,0.0029728000362714132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,128,0.0030378667016824085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,10240,0.03196799953778585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,8192,0.02810346682866414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,12288,0.037205334504445395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,7168,0.02502826650937398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,16384,0.04868053197860718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,65536,0.1850623925526937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,4096,0.013473066687583923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,3584,0.011973333358764649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,5120,0.01606186628341675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,3072,0.010917333761850993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,6144,0.022025599082310995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,2048,0.008318933347860973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,2560,0.01062506635983785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,1536,0.007271466652552287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,256,0.0033173332611719764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,1024,0.005046399931112925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,512,0.003684266656637192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,768,0.0040778666734695435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,128,0.0030261332790056865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,32,0.0030741333961486817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,64,0.002958933264017105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,8192,0.02376426657040914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,10240,0.028500266869862872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,12288,0.03356906572977702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,16384,0.040755200386047366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,7168,0.021415466070175172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,65536,0.16211199760437012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,5120,0.014282666643460593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,3584,0.010962133606274922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,6144,0.017029333114624023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,4096,0.011896533767382304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,3072,0.009925333658854167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,2560,0.009708799918492635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,1536,0.006781866649786632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,2048,0.007842133442560833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,1024,0.004781866570313772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,512,0.003689600030581156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,768,0.004073599974314371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,128,0.003125333289305369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,256,0.0032458665470282235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,64,0.002865066627661387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,32,0.003054933249950409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,7168,0.01680533289909363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,8192,0.020759467283884683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,12288,0.0283135990301768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,10240,0.02467199961344401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,65536,0.14331413904825846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,16384,0.03502613306045532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,6144,0.014905599753061929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,5120,0.012910933295885722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,4096,0.01076479951540629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,3584,0.009992532928784688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,3072,0.010082133611043294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,2560,0.008235733211040496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,2048,0.007186133166154225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,1024,0.004339199761549632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,1536,0.005961599946022034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,768,0.004023466755946477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,512,0.0036821333070596062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,128,0.0030975999931494398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,256,0.0032511999209721885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,64,0.002932266642649968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,32,0.002863999952872594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,7168,0.015904000401496886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,8192,0.017349332571029663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,65536,0.1217471996943156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,16384,0.02927680015563965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,12288,0.023707733551661173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,10240,0.020798933506011964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,6144,0.015490133563677469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,5120,0.012627200285593668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,3072,0.008460799853007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,4096,0.012226133545239767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,3584,0.0090421328941981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,2560,0.0075882668296496075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,2048,0.0067221333583196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,1536,0.005092266698678335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,1024,0.004226133227348328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,512,0.003568000098069509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,768,0.0039264000952243805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,128,0.002922666569550832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,64,0.0029397333661715193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,256,0.003160533308982849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,32,0.002977066735426585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,65536,0.09876266320546469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,7168,0.01307413379351298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,10240,0.016695467631022136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,8192,0.014266666769981385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,12288,0.01925440033276876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,16384,0.024125866095225015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,6144,0.012081066767374676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,5120,0.011145599683125814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,4096,0.010053333640098572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,3584,0.008518399794896443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,3072,0.007941333452860515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,1536,0.00489386667807897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,2048,0.00586346685886383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,2560,0.007189333438873291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,1024,0.004163199911514918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,256,0.00325546662012736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,768,0.00383146678407987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,512,0.0034858666360378264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,128,0.002899199972550074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,65536,0.07579306761423746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,32,0.002919466545184453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,64,0.0027893332143624624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,7168,0.01048533320426941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,10240,0.012734933694203695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,12288,0.014141866564750671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,8192,0.011051733295122783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,16384,0.01772480010986328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,6144,0.010583466291427613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,5120,0.009717333316802978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,4096,0.008772266904513042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,3584,0.007846400141716003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,3072,0.0074314668774604796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,1536,0.0048213332891464235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,2560,0.006237866481145223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,2048,0.005532800157864889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,1024,0.004166399935881296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,768,0.003806933263937632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,256,0.00323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,512,0.0034730667869249977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,128,0.0029685333371162414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,64,0.002872533351182938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,32,0.0029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,16384,0.014827733238538107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,65536,0.05288639863332113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,12288,0.012485333283742269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,6144,0.009168000022570292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,8192,0.010238933563232421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,10240,0.011549866199493409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,7168,0.009763200084368389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,3584,0.0075328002373377485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,5120,0.008473599950472515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,4096,0.007156266768773396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,65536,0.04546133279800415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,3072,0.006840533514817555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,2560,0.006166400015354156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,2048,0.005480533341566721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,1536,0.004772266745567322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,768,0.0037674665451049804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,1024,0.004335999985535939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,512,0.0034730667869249977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,32,0.002794666588306427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,64,0.0028042666614055633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,256,0.0031082667410373688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,128,0.002942933390537898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,10240,0.00940053363641103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,12288,0.01046399970849355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,8192,0.008582400282224019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,16384,0.011622400085131327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,7168,0.007844266792138416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,6144,0.007121066749095917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,5120,0.007322666545708974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,65536,0.034119466940561935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,4096,0.006667733192443848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,3584,0.007117866476376851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,2560,0.006077866752942403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,2048,0.005398400127887726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,3072,0.006785066425800323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,1536,0.00477866679430008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,1024,0.004070399949947992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,768,0.003719466676314672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,512,0.0033781332274278007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,256,0.003101866692304611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,32,0.0028938665986061097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,128,0.002921599894762039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,64,0.0029418667157491045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,16384,0.008866133292516072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,12288,0.007537066439787547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,8192,0.0069365332523981735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,10240,0.007358933488527934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,65536,0.023336533705393472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,7168,0.006820266445477803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,6144,0.006592000027497609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,5120,0.007009066641330719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,4096,0.006567466755708058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,3584,0.007036800185839335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,2560,0.006038400034109751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,3072,0.006739200154940288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,1536,0.0047775998711586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,2048,0.0053845331072807315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,1024,0.004048000027736028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,768,0.0037119999527931214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,512,0.0035957333942254386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,256,0.0031178665657838186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,128,0.0029002666473388673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,64,0.0029386666913827257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,32,0.002780800064404805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,16384,0.006750933329264323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,65536,0.013809067010879517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,12288,0.006684799989064534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,10240,0.006839466591676076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,8192,0.006833066542943318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,7168,0.0066453332702318835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,5120,0.006832000116507213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,6144,0.006448000172773997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,4096,0.006375466783841451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,3584,0.006949333349863689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,3072,0.006540800134340922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,2560,0.006044800082842508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,2048,0.005371733506520589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,1536,0.00469760000705719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,768,0.003688533355792364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,1024,0.004014933357636133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,512,0.0035861333211263022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,256,0.0030400000512599947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,128,0.0029504001140594482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,64,0.0028959999481836954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,32,0.002726399898529053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,16384,0.00660693347454071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,12288,0.0065642664829889934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,10240,0.006742399930953979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,65536,0.013130666812260947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,6144,0.006356266637643178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,8192,0.006694399813810985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,7168,0.00653546651204427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,5120,0.0067552000284194945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,3584,0.0067669332027435304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,4096,0.006316799918810527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,3072,0.006459733347098033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,2560,0.006042666733264923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,1536,0.004794666667779287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,2048,0.005356800059477488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,768,0.003623466690381368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,256,0.0029557332396507262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,512,0.003525333354870478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,1024,0.004003199934959412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,128,0.002903466671705246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,64,0.0028575999041398365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,32,0.0027413333455721537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,16384,0.006605866551399231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,12288,0.006498133142789205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,10240,0.006743466854095459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,65536,0.01167146662871043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,8192,0.006613333523273468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,6144,0.006331733365853627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,7168,0.006477866570154827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,5120,0.006625066697597504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,4096,0.006163200239340464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,3072,0.006242133180300395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,3584,0.006701866785685222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,2560,0.0059562668204307554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,2048,0.005312000215053558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,1024,0.004008533308903376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,768,0.0038592000802357995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,1536,0.004863999783992767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,512,0.0033557333052158357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,128,0.0028010666370391845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,256,0.002995199958483378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,32,0.0026986666023731233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,64,0.0027232001225153605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,5120,0.22959359486897787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,6144,0.2757962544759115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,7168,0.3218122800191244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,8192,0.3673365275065104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,4096,0.1876405398050944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,10240,0.45592320760091143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,3584,0.16305492719014486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,3072,0.14623680114746093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,2560,0.13818987210591632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,12288,0.5470080057779948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,1024,0.05841279824574789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,256,0.018642133474349974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,1536,0.08555520375569661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,128,0.009864532947540283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,512,0.028823467095692952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,2048,0.09860266844431559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,768,0.04599039951960246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,64,0.007435733576615651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,32,0.005764266848564148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,8192,0.09107200304667154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,7168,0.07971519629160563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,10240,0.11269013086954753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,12288,0.13366079330444336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,16384,0.7154794692993164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,5120,0.05871466795603434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,6144,0.06998613675435385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,16384,0.1763583978017171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,3072,0.04268480141957601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,4096,0.04826773405075073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,3584,0.04896639982859294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,1024,0.014613333344459533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,768,0.01157866617043813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,2048,0.026125866174697875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,2560,0.03150293429692586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,512,0.008739200234413148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,1536,0.023462400833765665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,256,0.005993600189685822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,64,0.0035306667288144433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,32,0.003647999962170919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,128,0.0038602667550245917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,7168,0.06506773233413696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,8192,0.0746997356414795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,10240,0.08724479675292969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,12288,0.10394986470540364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,16384,0.14516372680664064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,4096,0.03806186517079671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,5120,0.04736959934234619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,3584,0.03344320058822632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,6144,0.056385068098704014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,1536,0.016438399751981102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,1024,0.012079999844233195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,3072,0.031202133496602374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,2560,0.025116799275080363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,2048,0.021539199352264404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,768,0.009735467036565144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,512,0.00764160007238388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,256,0.004130133241415024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,128,0.0036330667634805044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,64,0.003335466732581457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,32,0.0034304000437259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,8192,0.05894080003102621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,10240,0.07321386337280274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,12288,0.08638613224029541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,16384,0.11379093329111736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,7168,0.05204693476359049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,6144,0.04581439892450968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,5120,0.03863786856333415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,4096,0.031856000423431396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,3584,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,2560,0.02139520049095154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,3072,0.02492799957593282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,1024,0.010204799969991048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,1536,0.013940266768137612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,2048,0.017783466974894205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,768,0.00851093331972758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,512,0.006883200009663899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,256,0.0038730666041374207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,65536,0.7095765431722005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,64,0.0032448001205921174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,128,0.0034858666360378264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,32,0.003335466732581457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,8192,0.04721813201904297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,65536,0.5525450388590495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,10240,0.05811306635538736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,12288,0.06906346480051676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,16384,0.10537919998168946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,7168,0.041980799039204916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,6144,0.042770131429036455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,5120,0.031300266583760575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,4096,0.02599253257115682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,3584,0.02624959945678711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,3072,0.020351999998092653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,1536,0.011462400356928509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,2048,0.014327466487884521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,768,0.007331199944019318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,2560,0.019808000326156615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,1024,0.008657067020734151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,128,0.0035146666069825493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,512,0.005861333509286245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,256,0.003774933268626531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,32,0.0032992000381151833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,64,0.0032853332658608755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,65536,0.45963093439737956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,8192,0.04193386634190877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,10240,0.05181013345718384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,12288,0.06155626773834229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,7168,0.04295039971669515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,16384,0.08549439907073975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,6144,0.033453865845998125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,5120,0.028220800558725993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,4096,0.023350399732589722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,3072,0.019078399737675986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,3584,0.020941867431004842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,2048,0.012941867113113403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,1536,0.010698666175206501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,2560,0.015887999534606935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,1024,0.008148266871770223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,768,0.007639466722806294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,65536,0.3634645462036133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,512,0.004533333579699198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,256,0.0035135999321937563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,64,0.0031221332649389905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,128,0.0032810665667057036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,32,0.0031871999303499854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,7168,0.03425600131352742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,8192,0.03913493156433105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,10240,0.0470197319984436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,12288,0.05608426729838053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,16384,0.07906880378723144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,4096,0.021617066860198975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,5120,0.02555519938468933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,3584,0.019341866175333657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,3072,0.01702293356259664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,6144,0.03032533327738444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,2560,0.014630400141080222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,65536,0.3414880116780599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,1536,0.010105599959691364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,2048,0.012488533059755962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,1024,0.007705600063006084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,768,0.006495999793211619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,256,0.0035125332574049628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,512,0.00429013321797053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,128,0.003234133372704188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,32,0.003091199944416682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,64,0.0030975999931494398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,8192,0.03131626645723979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,7168,0.02801919976870219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,12288,0.045067731539408365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,16384,0.06409706672032675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,10240,0.03838293155034383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,3072,0.014061866203943887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,3584,0.01594986617565155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,5120,0.02138239940007528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,4096,0.01764799952507019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,6144,0.02505279978116353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,2048,0.010272000233332317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,65536,0.27249813079833984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,2560,0.012070399522781373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,1536,0.008709333340326945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,1024,0.006829866766929626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,768,0.005257600049177805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,512,0.003869866579771042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,256,0.0033770665526390077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,128,0.00323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,32,0.0030432000756263735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,64,0.0030303999781608583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,12288,0.04040960073471069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,10240,0.03456639846165975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,8192,0.028807467222213744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,7168,0.026169600089391072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,16384,0.05035306612650553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,3072,0.011797333757082622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,3584,0.013171199957529703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,6144,0.021065600713094077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,4096,0.014680533607800802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,5120,0.017939200003941856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,65536,0.22713813781738282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,2560,0.010333866874376933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,1536,0.0075530668099721265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,2048,0.008905599514643352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,1024,0.006016000111897787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,768,0.004295466840267182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,128,0.0031136001149813337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,512,0.0037151999771595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,256,0.003336533407370249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,64,0.0029909332593282064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,32,0.0030293333033720653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,10240,0.031325866778691605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,7168,0.023829332987467446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,8192,0.026401066780090333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,16384,0.04599786599477132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,12288,0.03642880121866862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,65536,0.18213760058085124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,3584,0.012001066406567892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,4096,0.013106133540471396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,6144,0.01895893414815267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,5120,0.015946666399637856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,3072,0.010870400071144103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,2560,0.010523733496665955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,2048,0.008515200018882752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,1024,0.005070933202902476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,1536,0.007230933507283528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,768,0.003992533435424169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,128,0.0031178665657838186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,512,0.003668266783157984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,256,0.0032831999162832894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,64,0.003005866706371307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,32,0.002915200094381968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,10240,0.028552534182866414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,8192,0.024060799678166708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,12288,0.03337279955546062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,16384,0.040725334485371904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,7168,0.02220373352368673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,65536,0.16920960744222005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,5120,0.01667413314183553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,3584,0.010857599973678588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,3072,0.009873066345850627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,4096,0.01176533301671346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,6144,0.019734400510787963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,2560,0.009731200337409974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,2048,0.007762133578459422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,1536,0.00676800012588501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,512,0.0036138666172822317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,1024,0.004689066608746847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,768,0.003990400085846583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,256,0.003253333270549774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,64,0.002959999938805898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,128,0.0030794667700926462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,32,0.002924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,10240,0.024069333076477052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,8192,0.020409599939982096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,12288,0.02775146762530009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,65536,0.14082560539245606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,7168,0.018548266092936198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,16384,0.034593065579732254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,5120,0.014541866381963095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,6144,0.016711467504501344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,4096,0.012654933333396911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,3584,0.012928000092506409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,3072,0.009010133147239686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,2560,0.008254933357238769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,1536,0.006266666452089946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,2048,0.007155199845631917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,768,0.003978666663169861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,1024,0.004282666742801667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,512,0.00360000009338061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,128,0.003099733342727025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,64,0.0029397333661715193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,256,0.003234133372704188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,32,0.002919466545184453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,65536,0.11950826644897461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,10240,0.020434133211771646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,12288,0.02345386743545532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,7168,0.015568000078201295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,8192,0.01709653337796529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,16384,0.02874559958775838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,4096,0.011265066266059876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,5120,0.012562132875124612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,6144,0.014994133512179056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,3584,0.010539733370145162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,2560,0.007537066439787547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,3072,0.008436266581217449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,2048,0.006666666766007741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,1536,0.005256533126036326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,768,0.0038463999827702843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,1024,0.004221866528193155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,512,0.003443199892838796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,128,0.0030229332546393077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,256,0.0032330666979153953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,64,0.00288426677385966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,32,0.0028714666763941447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,65536,0.09676480293273926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,8192,0.013939199844996133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,10240,0.016552533706029257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,12288,0.01883093317349752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,16384,0.02347093423207601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,7168,0.013111467162768045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,5120,0.010908800363540649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,6144,0.012146133184432983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,4096,0.0092031995455424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,3584,0.008533333738644917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,2048,0.005804799993832906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,2560,0.006819200019041698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,3072,0.007934933404127757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,1536,0.004887466629346212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,512,0.0035573333501815797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,768,0.0039018665750821433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,1024,0.0041461333632469176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,256,0.00314026673634847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,128,0.002867199977238973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,65536,0.07393706639607747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,64,0.0028959999481836954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,32,0.0028949332733949023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,6144,0.009689600268999735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,12288,0.01421440045038859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,10240,0.012853333353996277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,8192,0.011291733384132386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,7168,0.010731732845306397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,16384,0.017384533087412515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,5120,0.009986133376757304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,4096,0.009054932991663616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,3584,0.008569600184758504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,2560,0.006597333153088887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,3072,0.007334400216738384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,1536,0.004867200056711833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,2048,0.005545599758625031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,65536,0.0543509324391683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,768,0.003828266759713491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,1024,0.00415040006240209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,128,0.003014400104681651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,512,0.0034933333595593774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,256,0.0031541332602500914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,32,0.002922666569550832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,64,0.0028319999575614927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,16384,0.014419200023015341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,12288,0.012039466698964437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,10240,0.01141439974308014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,7168,0.010732799768447876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,8192,0.011381333072980244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,6144,0.010053333640098572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,3072,0.006887466708819072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,4096,0.00795839975277583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,5120,0.009226666887601216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,3584,0.00739519993464152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,65536,0.04423786799112956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,2560,0.00613013356924057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,2048,0.005426133175690969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,1536,0.004783999919891357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,1024,0.004113066693147024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,768,0.0037674665451049804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,512,0.0034186666210492453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,32,0.0028746667007605235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,256,0.003058133274316788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,128,0.0029738667110602063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,64,0.0028010666370391845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,10240,0.010563199718793232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,12288,0.011443199714024861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,16384,0.011401599645614624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,8192,0.009454933802286784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,5120,0.007429333527882893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,7168,0.008599467078844706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,6144,0.008201600114504496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,4096,0.006682666639486949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,3584,0.007035733262697856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,65536,0.03301013310750325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,2560,0.006049066781997681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,3072,0.006742399930953979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,2048,0.005399466554323832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,1536,0.0047199999292691554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,256,0.0030986666679382324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,768,0.0036906667053699495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,1024,0.004067199925581614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,512,0.0033941333492596946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,128,0.002856533229351044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,64,0.002792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,32,0.002755200117826462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,10240,0.007567999760309856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,16384,0.009830400347709656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,12288,0.008523733417193095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,8192,0.006951466699441274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,65536,0.022445867458979286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,5120,0.0068896000583966565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,7168,0.006758399804433187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,6144,0.006505600114663441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,4096,0.006506666541099548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,3584,0.007041066884994507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,3072,0.006687999765078227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,2048,0.005357866485913595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,2560,0.006045866509278615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,1536,0.004711466530958811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,768,0.003688533355792364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,1024,0.004073599974314371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,512,0.003356799980004629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,256,0.002980266759792964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,128,0.0029535998900731405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,64,0.002979200085004171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,32,0.0028149334092934927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,7168,0.00660693347454071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,8192,0.006790400048096975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,12288,0.006677333513895671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,10240,0.00688213308652242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,16384,0.0066997334361076355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,65536,0.013802666465441385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,5120,0.006832000116507213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,4096,0.006479999919732411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,6144,0.006517333288987477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,3584,0.00693333347638448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,3072,0.0065536002318064375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,2048,0.0053151999910672505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,2560,0.0060032000144322716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,1536,0.004702933132648468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,1024,0.004054400076468786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,512,0.0033514666060606635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,768,0.0037087999284267426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,256,0.0029557332396507262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,128,0.0029098667204380036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,32,0.0028010666370391845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,64,0.002757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,7168,0.006549333532651265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,16384,0.006623999774456024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,12288,0.006551466882228851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,10240,0.006730666756629944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,65536,0.012611200412114462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,8192,0.006614399949709575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,6144,0.006415999929110209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,5120,0.006811733543872833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,4096,0.006355200211207073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,3072,0.0065301333864529925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,2048,0.005302399893601735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,3584,0.006874666611353557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,2560,0.005965866645177205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,1024,0.003990400085846583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,1536,0.004647466540336609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,32,0.002765866617361705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,768,0.003655466685692469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,512,0.003268266717592875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,256,0.0030069333811601003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,128,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,64,0.0027232001225153605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,16384,0.006577066580454509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,12288,0.006449066599210103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,65536,0.011618133385976155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,10240,0.0066442668437957765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,8192,0.006603733201821645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,7168,0.006481066842873891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,6144,0.006388266881306966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,5120,0.006733866532643636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,4096,0.006358399987220764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,2560,0.005967999994754791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,1024,0.003962666789690653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,3072,0.006355200211207073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,3584,0.006734933455785115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,2048,0.005310933291912079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,1536,0.004619733492533366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,256,0.003028266628583272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,768,0.0036085332433382668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,512,0.0033215999603271483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,128,0.00278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,32,0.0027253332237402597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,64,0.0027093333502610523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,5120,0.23045013745625814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,6144,0.27379945119222004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,7168,0.31704746882120766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,8192,0.3652234713236491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,4096,0.18659520149230957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,3584,0.16179946263631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,10240,0.45422932306925456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,3072,0.14392746289571126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,2560,0.1368672053019206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,12288,0.5439210891723633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,768,0.044888532161712645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,1024,0.05757226546605428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,512,0.027636265754699706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,256,0.017864533265431724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,1536,0.0842858632405599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,128,0.009340799848238627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,64,0.006954666475454967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,2048,0.09844160079956055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,32,0.005256533126036326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,16384,0.7167669296264648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,7168,0.07971519629160563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,8192,0.09129707018534342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,10240,0.11145493189493816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,12288,0.1340789318084717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,6144,0.08124159971872966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,5120,0.05832639932632446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,16384,0.17633280754089356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,4096,0.047890134652455646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,3584,0.04200213352839152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,3072,0.04208213488260905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,2560,0.03128426671028137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,768,0.012665599584579468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,512,0.008664533495903015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,1024,0.014386133352915446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,1536,0.023179733753204347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,2048,0.025718400875727337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,256,0.005891199906667074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,128,0.0036917333801587426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,64,0.0033151999115943907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,32,0.0034783999125162757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,7168,0.0654698650042216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,8192,0.07493120034535726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,12288,0.1054357369740804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,10240,0.09170347054799398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,16384,0.1400757312774658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,4096,0.03761066595713298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,5120,0.04688959916432699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,3584,0.03442026774088542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,6144,0.05636159976323446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,3072,0.029959466060002642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,2560,0.02482773264249166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,1024,0.011865599950154623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,2048,0.021206400791803994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,1536,0.016721065839131674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,512,0.007369600236415863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,768,0.009660800298055012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,128,0.003504000107447306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,32,0.0032672000428040824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,256,0.004035199930270513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,64,0.0032287999987602235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,8192,0.05904106696446737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,10240,0.07062506675720215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,12288,0.08726826508839926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,16384,0.11432000001271565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,5120,0.038762664794921874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,7168,0.052262401580810545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,6144,0.04493120114008586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,4096,0.03281066616376241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,3584,0.02805759906768799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,3072,0.024771199623743693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,1536,0.013713066776593527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,2560,0.02084160049756368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,2048,0.017427200078964235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,1024,0.010103467106819152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,768,0.00844266712665558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,65536,0.7025450388590495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,512,0.006713599960009258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,128,0.0033344000577926634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,256,0.003684266656637192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,64,0.0032320000231266023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,32,0.0032543999453385672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,65536,0.5437386830647786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,8192,0.04701653321584066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,10240,0.057988266150156655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,12288,0.06896853446960449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,16384,0.08939200242360433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,6144,0.03692906697591146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,5120,0.031102933486302692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,7168,0.04815040032068889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,4096,0.025667200485865276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,1536,0.011340799927711486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,2560,0.017198934157689413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,3584,0.022850133975346885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,3072,0.020169599850972494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,2048,0.016242133577664693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,256,0.00365226666132609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,768,0.007285333176453908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,1024,0.008500267068545024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,512,0.005825066566467285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,128,0.0033429334561030067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,64,0.0031680000325044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,32,0.0032778667906920114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,65536,0.4550506591796875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,8192,0.04178666671117147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,10240,0.05162239869435629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,12288,0.061432532469431554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,7168,0.04269333283106486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,16384,0.07989546457926432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,5120,0.027910399436950683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,6144,0.032788266738255814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,3584,0.022457599639892578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,4096,0.023177599906921385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,3072,0.018334933121999106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,2048,0.012936533490816752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,2560,0.01574720044930776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,1536,0.010609066486358643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,1024,0.008061866462230682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,512,0.004401066899299621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,768,0.00684799998998642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,256,0.003469866762558619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,128,0.0032330666979153953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,32,0.003126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,64,0.0030048000315825146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,65536,0.36324586868286135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,10240,0.045525332291920975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,8192,0.039095465342203775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,7168,0.03901013135910034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,12288,0.05640319983164469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,16384,0.07111360232035319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,6144,0.02942933241526286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,3584,0.019228800137837728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,3072,0.016548267006874083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,4096,0.021512534221013388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,5120,0.02547840078671773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,1536,0.009753599762916565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,2560,0.014035200079282125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,2048,0.012308266758918763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,1024,0.0075434664885203045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,768,0.006351999938488007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,512,0.003853866706291834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,65536,0.32574612299601235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,256,0.0033610666791598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,128,0.003151999910672506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,32,0.003017599880695343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,64,0.0030069333811601003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,12288,0.04524266719818115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,10240,0.03841919898986816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,7168,0.027537065744400024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,8192,0.031172267595926922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,16384,0.06759253342946371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,6144,0.024730666478474935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,3072,0.013844266533851624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,3584,0.015851733088493348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,5120,0.021128533283869426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,4096,0.017364267508188883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,2560,0.012065066893895467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,2048,0.010206933816274006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,1024,0.006778666873772939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,1536,0.008685866991678875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,65536,0.27013867696126304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,512,0.0036960000793139136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,768,0.0052714665730794275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,256,0.0032458665470282235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,128,0.003126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,32,0.0029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,64,0.0029120000700155893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,7168,0.025496532519658405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,8192,0.028154667218526202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,12288,0.0396725336710612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,10240,0.03385813236236572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,16384,0.05018773476282755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,3584,0.013038933277130127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,3072,0.011822932958602905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,65536,0.22657279968261718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,4096,0.014734933773676554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,5120,0.017661867539087932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,6144,0.023000532388687135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,2560,0.010289067029953003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,1536,0.008355200290679932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,1024,0.005825066566467285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,2048,0.008993066350618998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,768,0.0042367999752362575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,128,0.0030634666482607523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,512,0.0035797332723935447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,256,0.0032117334504922234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,64,0.0028266665836175283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,32,0.002958933264017105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,10240,0.030635732412338256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,16384,0.04478933413823445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,12288,0.03567786614100139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,8192,0.02576106588045756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,65536,0.1811039924621582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,7168,0.023372799158096313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,3072,0.012150399883588155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,3584,0.011828266580899556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,4096,0.013050666451454163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,2560,0.00958293378353119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,5120,0.018318933248519898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,6144,0.020921599864959717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,2048,0.008413867155710856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,1536,0.00714026689529419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,512,0.003537066777547201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,1024,0.005072000126043955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,768,0.003942399968703588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,256,0.0032127998769283296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,32,0.0030026666820049284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,128,0.0030730667213598887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,64,0.0027935999135176343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,7168,0.021724800268809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,8192,0.023521065711975098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,12288,0.0326581339041392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,10240,0.027540266513824463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,16384,0.04147093296051026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,65536,0.16372693379720052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,3584,0.010909866293271382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,5120,0.016402133305867515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,4096,0.014782933394114175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,6144,0.01942080060640971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,3072,0.009858133395512898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,2048,0.007749333480993907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,2560,0.009705600142478944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,1536,0.006735999882221222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,1024,0.004465066889921824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,768,0.003908266623814901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,256,0.0032970666885375976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,64,0.0029109333952267963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,128,0.0029887999097506206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,512,0.0035317334036032355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,32,0.0028597332537174227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,12288,0.02738453348477681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,7168,0.018191999197006224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,8192,0.019886932770411172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,10240,0.02381333311398824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,65536,0.1400810718536377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,16384,0.0341813325881958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,6144,0.016221867005030314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,2560,0.00819946676492691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,5120,0.014478933811187745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,3584,0.011797333757082622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,4096,0.012345600128173827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,3072,0.011956266562143962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,1536,0.005996799965699514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,2048,0.0070826664566993715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,1024,0.004198400179545084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,768,0.003913599997758865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,256,0.0032298666735490165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,512,0.003538133452335993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,128,0.0030037333567937215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,64,0.0028629332780838014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,32,0.002917333443959554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,65536,0.1189802646636963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,10240,0.019926400979359944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,7168,0.015239466230074564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,8192,0.01665279964605967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,12288,0.022949333985646567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,16384,0.028155734141667683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,5120,0.012411733468373615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,6144,0.014877866705258688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,3584,0.01032426655292511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,4096,0.010985599954922994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,3072,0.009668266773223877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,2560,0.007531733314196269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,1024,0.00413226659099261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,2048,0.006726400057474773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,1536,0.0048213332891464235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,768,0.003773866593837738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,512,0.0034506666163603462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,128,0.0029845332105954488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,64,0.0028490667541821797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,256,0.003134933362404505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,32,0.00275093341867129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,65536,0.0955135981241862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,10240,0.016210132837295534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,7168,0.01304213305314382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,8192,0.013981866836547851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,12288,0.018548266092936198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,5120,0.010855467120806376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,16384,0.02304960091908773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,6144,0.0120469331741333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,4096,0.009842133522033692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,3584,0.009468799829483033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,2560,0.007047466437021891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,3072,0.008905599514643352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,1536,0.00480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,2048,0.005688533186912537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,512,0.0034304000437259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,1024,0.004098133246103922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,768,0.0037813333173592886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,65536,0.07302719751993815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,256,0.0032511999209721885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,128,0.0029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,32,0.0028778667251269023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,64,0.0028138667345046996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,12288,0.014110933740933737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,7168,0.010546132922172546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,10240,0.012955733140309653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,8192,0.01116480032602946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,16384,0.01721386710802714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,6144,0.009567999839782714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,4096,0.008307200173536937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,5120,0.009212799866994222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,3584,0.008151466647783916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,3072,0.007098666826883952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,1536,0.004724266628424326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,2560,0.0060810665289560955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,65536,0.053446400165557864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,2048,0.005406933526198069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,1024,0.004072533299525579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,256,0.0031370667119820913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,768,0.003722666700681051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,512,0.0034400001168251038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,64,0.0027200000981489818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,128,0.0029397333661715193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,32,0.0027776000400384264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,16384,0.014123732844988504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,12288,0.011949867010116577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,10240,0.01141866644223531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,8192,0.01009173293908437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,5120,0.008281599978605907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,7168,0.009477333227793375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,6144,0.008661333719889324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,4096,0.007162666817506154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,3584,0.0073728000124295544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,2560,0.006067200005054474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,65536,0.04292906522750854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,3072,0.00693333347638448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,2048,0.0053941334287325535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,1536,0.004731733103593191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,512,0.0033919999996821085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,1024,0.004075733323891958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,768,0.0037237333754698435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,128,0.0028917332490285236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,256,0.0030762667457262674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,64,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,32,0.0028064000109831494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,16384,0.011159466703732808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,12288,0.01013866662979126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,10240,0.0093450665473938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,8192,0.008589866757392883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,5120,0.007271466652552287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,65536,0.031702399253845215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,7168,0.00767146646976471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,6144,0.007402666906515758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,3584,0.00702400008837382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,4096,0.0065653334061304735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,3072,0.006697600086530049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,2560,0.006066133578618368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,2048,0.005357866485913595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,1536,0.004713599880536398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,1024,0.004041599979003271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,768,0.003692800054947535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,128,0.00297173336148262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,512,0.003369600077470144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,256,0.0030847998956839246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,64,0.002746666719516118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,32,0.0028277332584063213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,12288,0.007361066838105519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,16384,0.008429867029190064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,10240,0.00699946681658427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,7168,0.006684799989064534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,8192,0.0068341334660847975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,65536,0.02116159995396932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,3072,0.00665280024210612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,5120,0.006930133203665416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,6144,0.006528000036875407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,4096,0.0065184002121289565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,3584,0.0069482664267222095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,2560,0.006028800209363302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,1536,0.00469760000705719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,2048,0.005339733262856802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,1024,0.00403413325548172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,512,0.0033546666304270422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,256,0.0029738667110602063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,768,0.003685333331425985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,128,0.0028490667541821797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,64,0.0027903998891512555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,32,0.002809600035349528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,7168,0.006487466891606649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,16384,0.006694399813810985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,12288,0.0064735998709996535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,8192,0.006596266726652781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,10240,0.006687999765078227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,65536,0.013895466923713684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,5120,0.0066442668437957765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,4096,0.006333866715431213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,3584,0.006760533154010773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,3072,0.006423466900984447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,6144,0.006344533463319142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,2048,0.005312000215053558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,2560,0.006031999985376993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,1536,0.004677333434422811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,1024,0.004003199934959412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,768,0.0036544000109036768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,32,0.002810666710138321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,512,0.0033439998825391137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,256,0.003010133405526479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,128,0.0029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,64,0.002771199991305669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,12288,0.006495999793211619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,10240,0.006668800115585327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,16384,0.0065749332308769224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,65536,0.012541866302490235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,7168,0.006459733347098033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,8192,0.006614399949709575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,5120,0.006550399959087372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,6144,0.0062730665008227035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,4096,0.006163200239340464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,3584,0.006680533289909363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,1536,0.004637866715590159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,3072,0.006376533210277558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,2560,0.005962666869163513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,2048,0.005321600039800008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,1024,0.004010666658480962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,256,0.0030303999781608583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,768,0.003636266787846883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,512,0.0032853332658608755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,128,0.0028309332827727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,64,0.0027530667682488757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,16384,0.006562133133411407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,32,0.0027104000250498454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,12288,0.006841599941253662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,65536,0.012146133184432983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,10240,0.00664213349421819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,8192,0.006420266628265381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,6144,0.006140799820423126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,7168,0.006337066491444905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,5120,0.006487466891606649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,4096,0.006146133442719777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,2048,0.0053258667389551794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,3072,0.006295466423034668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,2560,0.005970133344332377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,3584,0.006678399940331777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,1536,0.004655999938646952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,512,0.0033610666791598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,1024,0.003962666789690653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,768,0.0036117332677046456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,256,0.0030400000512599947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,128,0.002757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,64,0.0027317332724730173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,32,0.0027615999182065325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,5120,0.22613867123921713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,6144,0.27217067082722984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,7168,0.31563520431518555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,8192,0.3641471862792969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,4096,0.18433067003885906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,10240,0.452396806081136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,3584,0.1605994701385498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,3072,0.14275840123494465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,2560,0.1358239968617757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,2048,0.09676693280537924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,1024,0.05664746761322022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,768,0.04392319917678833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,1536,0.08338773250579834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,256,0.016940800348917644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,12288,0.5470837275187175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,512,0.026687999566396076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,64,0.006401066482067108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,128,0.00906986693541209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,32,0.004701866706212362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,7168,0.07850666840871176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,8192,0.08998933633168539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,10240,0.11034986972808838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,16384,0.7129173278808594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,12288,0.1321013371149699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,3584,0.04830933411916097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,5120,0.05746666590372721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,16384,0.20700906117757162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,4096,0.04689919948577881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,6144,0.06876373291015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,3072,0.04203199942906698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,768,0.012894933422406515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,2560,0.03076159954071045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,1024,0.014203733205795288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,512,0.00848533312479655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,1536,0.022797866662343343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,2048,0.025293866793314617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,128,0.0035232000052928926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,256,0.00550186683734258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,64,0.0031829332311948144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,32,0.003257599969704946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,7168,0.060317866007486974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,8192,0.0697866678237915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,10240,0.08662827014923095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,12288,0.10682346820831298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,16384,0.13834452629089355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,3584,0.032681600252787275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,4096,0.03697173198064168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,5120,0.045254401365915936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,6144,0.05381866693496704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,1024,0.012119467059771221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,2560,0.024234666426976522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,3072,0.03224106629689534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,1536,0.01586026648680369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,2048,0.020167466004689535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,512,0.007420800129572551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,768,0.00965013305346171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,32,0.0032448001205921174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,128,0.0034506666163603462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,64,0.003218133250872294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,256,0.004020266731580098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,8192,0.05841600100199381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,10240,0.07229866981506347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,12288,0.08607359727223715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,16384,0.11279466946919758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,7168,0.05165013472239176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,6144,0.045201067129770914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,5120,0.03804373343785604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,4096,0.032363732655843094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,2048,0.017203199863433837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,2560,0.020836265881856282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,3072,0.024272000789642333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,3584,0.027723733584086103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,1536,0.013542399803797404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,65536,0.7058015823364258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,1024,0.010053333640098572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,768,0.008286933104197185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,512,0.006620799998442332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,256,0.0035957333942254386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,128,0.003302400062481562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,64,0.002993066608905792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,32,0.0030239999294281008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,8192,0.046428799629211426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,12288,0.06799466609954834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,10240,0.05705386797587077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,65536,0.5442837397257487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,16384,0.08858666419982911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,3584,0.022524799903233847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,5120,0.03064746658007304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,4096,0.028938666979471846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,7168,0.04115093151728312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,6144,0.036421334743499754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,1024,0.008569600184758504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,1536,0.011255466938018798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,3072,0.019853866100311278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,2560,0.017086933056513466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,768,0.007167999943097432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,2048,0.014099199573198953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,512,0.005760000149408976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,64,0.003068800022204717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,256,0.0035807999471823373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,128,0.0031669333577156065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,32,0.0031456001102924346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,65536,0.45303252538045247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,8192,0.04137493371963501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,10240,0.05457280079523722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,12288,0.06419626474380494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,7168,0.042507731914520265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,16384,0.08215786616007487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,4096,0.022808533906936646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,5120,0.027612799406051637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,6144,0.03243733247121175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,3584,0.02042133410771688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,3072,0.01811306675275167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,2048,0.012774399916330972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,2560,0.015599999825159708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,768,0.006810666620731353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,1024,0.008959999680519104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,1536,0.010533333818117777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,512,0.004232533276081085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,256,0.0034111998975276947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,128,0.003110400090614955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,65536,0.35891841252644857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,64,0.002932266642649968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,32,0.0029824001093705496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,8192,0.03810880184173584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,12288,0.052988799413045254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,7168,0.03352320194244385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,10240,0.05306346813837687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,16384,0.06972373326619466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,5120,0.02449280023574829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,3584,0.018860799074172974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,4096,0.02104533314704895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,6144,0.02946453293164571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,3072,0.016190933187802632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,2560,0.014028799533843995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,1024,0.007447466750939687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,768,0.006233599781990051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,2048,0.012179199854532878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,1536,0.009880533814430237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,512,0.0037770666182041167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,128,0.003014400104681651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,65536,0.3069525400797526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,256,0.003305600086847941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,64,0.002946133414904277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,32,0.002958933264017105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,7168,0.027245867252349853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,10240,0.03800959984461467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,8192,0.030882134040196733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,12288,0.051582932472229004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,16384,0.05830613374710083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,3072,0.013706666231155396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,3584,0.01554026703039805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,4096,0.017398399114608765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,5120,0.020898133516311646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,2560,0.011865599950154623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,6144,0.024241065979003905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,768,0.005449600021044413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,2048,0.010305066903432209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,1536,0.008611200253168742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,1024,0.006718933085600535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,65536,0.2679295857747396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,512,0.003587199995915095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,256,0.0032330666979153953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,128,0.003047466774781545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,32,0.0028597332537174227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,64,0.0028543998797734577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,12288,0.03906666835149129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,7168,0.02507839997609456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,8192,0.027755733331044512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,10240,0.033174399534861246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,16384,0.04940906763076782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,6144,0.02253440022468567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,3584,0.015064533551534018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,3072,0.0116757333278656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,4096,0.016356266538302102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,5120,0.01762346625328064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,2560,0.01027413308620453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,65536,0.22338560422261558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,2048,0.008855467041333515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,512,0.0036458666125933326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,1536,0.007597866654396057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,1024,0.00595413347085317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,768,0.004092800120512644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,256,0.003357866654793421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,64,0.0029056000212828318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,128,0.002993066608905792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,32,0.0030016000072161358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,10240,0.030057599147160847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,8192,0.02499413291613261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,12288,0.03456213474273682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,7168,0.022925867636998495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,16384,0.04394666751225789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,65536,0.1776799996693929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,4096,0.015123200416564942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,6144,0.020504534244537354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,5120,0.017816533644994102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,2048,0.00842133363087972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,3584,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,3072,0.010670933127403259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,2560,0.010491733749707539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,1536,0.0071839998165766404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,768,0.004126933217048645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,1024,0.004839466512203216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,128,0.0029557332396507262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,512,0.0035232000052928926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,256,0.003201066702604294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,64,0.0029130667448043824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,32,0.0029813334345817565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,7168,0.020803199211756388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,8192,0.02295253276824951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,10240,0.027476267019907637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,12288,0.031030400594075518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,16384,0.03951679865519206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,65536,0.16275733311971027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,5120,0.016153599818547568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,4096,0.014351999759674073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,6144,0.018606932957967122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,3072,0.0120469331741333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,3584,0.01325866679350535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,2048,0.0076799998680750535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,2560,0.009710933764775593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,256,0.0032298666735490165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,1536,0.006694399813810985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,1024,0.00450133333603541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,768,0.0038560000558694207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,512,0.00347626656293869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,128,0.0028917332490285236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,64,0.0028938665986061097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,32,0.002880000074704488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,8192,0.019700266917546592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,65536,0.13835840225219725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,10240,0.02327573299407959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,16384,0.03345280090967814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,12288,0.026580266157786053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,6144,0.015868799885114034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,7168,0.01793173352877299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,2560,0.008226133386294047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,5120,0.015646933515866598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,4096,0.012437333663304646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,3584,0.011586133639017742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,3072,0.010620799660682679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,2048,0.007028266787528992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,1536,0.006153599917888641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,1024,0.004164266586303711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,768,0.0038261334101359046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,512,0.0034677334129810332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,64,0.002865066627661387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,256,0.0032159999012947083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,128,0.0030026666820049284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,32,0.002883200099070867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,65536,0.11650239626566569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,12288,0.02228800058364868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,7168,0.015016532937685647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,16384,0.027559467156728107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,8192,0.016269866625467935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,10240,0.019517866770426433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,6144,0.014357333381970724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,5120,0.012312533458073933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,4096,0.01083733340104421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,3584,0.010181333621342976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,3072,0.009611733754475911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,1536,0.005120000243186951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,2560,0.007477333148320516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,2048,0.006676266590754191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,1024,0.004090666770935059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,768,0.0037450666228930154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,256,0.00317546675602595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,512,0.0034186666210492453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,64,0.0027978666126728057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,128,0.0029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,32,0.0028021333118279776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,65536,0.09304107030232747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,10240,0.01597653329372406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,8192,0.013831466436386108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,7168,0.012925866246223449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,16384,0.022362667322158813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,6144,0.012919466694196066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,12288,0.018363734086354576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,5120,0.010572800040245056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,3584,0.009244799613952637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,3072,0.008782933155695598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,4096,0.009784533580144247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,1536,0.004739200075467428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,2560,0.007038933535416921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,2048,0.005749333401521047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,256,0.0031744000812371576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,768,0.003802666564782461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,1024,0.004045866678158442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,512,0.003399466723203659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,65536,0.07233920097351074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,128,0.002889599899450938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,64,0.002713600049416224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,32,0.002792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,6144,0.009467732906341553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,10240,0.012651733557383218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,12288,0.013760000467300415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,8192,0.011100799838701884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,16384,0.01688213348388672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,7168,0.010491733749707539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,5120,0.009153067072232565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,4096,0.008306133250395458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,3072,0.007117866476376851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,3584,0.007586133480072021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,2560,0.006134399771690368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,1536,0.004748799900213877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,2048,0.005416533350944519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,65536,0.04934080044428508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,1024,0.004098133246103922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,768,0.003701333453257879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,128,0.0029738667110602063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,256,0.0030517332255840302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,512,0.0033759998778502146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,32,0.00279573326309522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,64,0.002765866617361705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,12288,0.011770666639010111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,16384,0.014083199699719749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,7168,0.009300266702969868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,10240,0.01115733285744985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,8192,0.010009599725405376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,6144,0.008483200271924337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,3584,0.007292800148328145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,5120,0.008075733482837678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,4096,0.006967466572920482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,3072,0.006838400165239971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,65536,0.0392576018969218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,2560,0.006054399907588959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,1536,0.00469760000705719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,2048,0.0053951998551686605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,1024,0.004060799876848856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,768,0.0037087999284267426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,512,0.0033759998778502146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,64,0.0027872001131375628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,256,0.0030165334542592366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,128,0.0029109333952267963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,32,0.002784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,10240,0.009315199653307597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,16384,0.011218133568763732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,12288,0.009910399715105694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,8192,0.008633599678675333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,5120,0.006807466844717662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,7168,0.007494399944941203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,65536,0.028911999861399335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,6144,0.00692799985408783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,4096,0.0065184002121289565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,3584,0.007042133311430614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,3072,0.006714666883150737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,2560,0.006031999985376993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,1536,0.004730666677157084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,2048,0.005362133185068766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,1024,0.004084266722202301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,128,0.0029546665648619336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,768,0.003731200098991394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,512,0.0033728001018365227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,256,0.0029696000119050344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,64,0.0028053333361943563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,16384,0.008251733581225077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,32,0.0027477333943049112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,12288,0.0071839998165766404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,10240,0.007029333213965099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,8192,0.006592000027497609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,6144,0.0064074665307998655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,65536,0.017819732427597046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,7168,0.0065536002318064375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,5120,0.006867200136184692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,3584,0.00687360018491745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,3072,0.006600533425807953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,4096,0.00643093337615331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,2560,0.0060138667623202005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,1536,0.00472320020198822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,2048,0.005382399757703145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,128,0.0029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,768,0.003664000084002813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,512,0.0033141332368055976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,1024,0.003992533435424169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,256,0.0029567999144395193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,64,0.002791466563940048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,32,0.0028170667588710784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,65536,0.012427733341852824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,12288,0.006704000135262807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,16384,0.006554666658242543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,8192,0.00662720004717509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,10240,0.006718933085600535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,7168,0.0065194666385650635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,5120,0.00654720018307368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,6144,0.006296533346176148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,4096,0.006188799937566122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,3584,0.006649599969387054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,3072,0.006318933268388112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,1536,0.004537599782148996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,2560,0.005773866673310598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,2048,0.00518506666024526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,1024,0.0039658665657043455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,768,0.0036138666172822317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,128,0.0028959999481836954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,512,0.0032799998919169106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,256,0.0029834667841593427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,32,0.002757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,64,0.002721066772937775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,12288,0.006292266647020976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,16384,0.006451199948787689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,10240,0.006444799900054932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,8192,0.006427733103434245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,65536,0.009806933005650838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,7168,0.006258133550484974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,5120,0.006514133512973785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,6144,0.006243200103441874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,4096,0.0061589335401852924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,3584,0.006534400085608165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,3072,0.006275199850400289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,1536,0.004452266792456309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,2560,0.0056874667604764305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,2048,0.005048533280690512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,1024,0.003822933385769526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,768,0.003565866748491923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,512,0.003218133250872294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,128,0.0028170667588710784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,256,0.0029781334102153777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,32,0.002700799951950709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,64,0.0027253332237402597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,10240,0.006378666559855144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,16384,0.006607999900976817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,12288,0.006460799773534138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,65536,0.008077866832415263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,8192,0.0063381334145863845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,7168,0.0061951999862988796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,5120,0.0064181332786877945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,6144,0.006093866626421611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,4096,0.006085333228111267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,3584,0.006491733094056447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,3072,0.006126933296521505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,2560,0.0056970665852228795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,1536,0.0044501334428787235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,2048,0.005015466610590617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,1024,0.003866666555404663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,768,0.0035445332527160645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,512,0.0032426667710145317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,64,0.002700799951950709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,128,0.0028021333118279776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,32,0.0027061333258946735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,256,0.002917333443959554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,5120,0.22752532958984376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,6144,0.2714783986409505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,7168,0.3146901448567708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,8192,0.3622687975565592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,3584,0.16018452644348144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,4096,0.18393066724141438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,10240,0.45119466781616213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,2560,0.13595412572224935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,3072,0.1415509382883708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,1024,0.05581440130869547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,768,0.03752640088399251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,12288,0.5428330739339192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,2048,0.10467627048492431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,512,0.029765333731969195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,1536,0.07914986610412597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,128,0.008892800410588582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,256,0.014516266187032065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,64,0.006817066669464111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,32,0.00444160004456838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,7168,0.07866026560465494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,8192,0.0892959992090861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,10240,0.1102176030476888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,12288,0.13260266780853272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,16384,0.7142762502034505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,6144,0.06866559982299805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,5120,0.05736213525136312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,16384,0.17642347017923993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,3584,0.047884798049926756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,4096,0.04655040105183919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,3072,0.04193280140558879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,768,0.012898133198420206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,1024,0.014148267110188803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,512,0.008483200271924337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,2560,0.030530132850011188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,2048,0.02502826650937398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,128,0.00347626656293869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,1536,0.022637865940729775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,256,0.005756799876689911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,32,0.003270400067170461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,64,0.0031744000812371576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,8192,0.0696938673655192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,7168,0.061045332749684655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,10240,0.08618026574452718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,12288,0.10600639979044597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,16384,0.13784106572469074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,5120,0.04497493505477905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,6144,0.05497173468271891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,4096,0.03657066822052002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,3584,0.031915734211603805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,3072,0.0319541335105896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,2048,0.02042133410771688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,1536,0.01564906636873881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,1024,0.011447466413180033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,2560,0.02396799921989441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,512,0.007566933333873749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,768,0.00962559978167216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,256,0.00384853333234787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,128,0.0034613333642482757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,64,0.003190399954716364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,32,0.0031957333286603295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,8192,0.05821866591771444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,10240,0.07221972942352295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,12288,0.08344213167826334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,16384,0.11328106721242268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,7168,0.0515989343325297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,6144,0.045273598035176596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,5120,0.037036800384521486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,3584,0.027463465929031372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,4096,0.03226666649182637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,1536,0.013353600104649862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,2048,0.016936532656351724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,3072,0.024267733097076416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,2560,0.020376533269882202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,768,0.008213333288828532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,1024,0.009893332918485005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,512,0.006524799764156342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,128,0.003293866664171219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,256,0.0035434665779272715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,64,0.003058133274316788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,65536,0.7017920176188152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,32,0.00297173336148262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,65536,0.5368832270304362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,8192,0.0460970679918925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,10240,0.0664469321568807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,12288,0.06753280162811279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,16384,0.08878719806671143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,7168,0.04768319924672444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,6144,0.03629973332087199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,4096,0.02488320072491964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,5120,0.03036053379376729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,3584,0.022342399756113688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,3072,0.01960106690724691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,1536,0.011252267162005107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,2048,0.013974400361378989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,2560,0.016858667135238647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,1024,0.008457600076993307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,768,0.007192533214886982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,65536,0.4521418571472168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,512,0.005764266848564148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,256,0.0034186666210492453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,128,0.0032117334504922234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,64,0.003032533327738444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,32,0.0029098667204380036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,7168,0.040403199195861814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,8192,0.044308265050252274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,10240,0.05366506576538086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,12288,0.06335146824518839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,16384,0.08127360343933106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,3584,0.02026559909184774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,6144,0.034942932923634845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,5120,0.030179200569788615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,4096,0.02255786657333374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,3072,0.017938133080800375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,65536,0.35462719599405923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,2048,0.01285653313000997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,1024,0.00798933357000351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,1536,0.010476799805959065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,2560,0.015375999609629312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,768,0.006754133105278015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,512,0.004189866781234741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,256,0.0032799998919169106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,128,0.003151999910672506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,32,0.0029450667401154833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,64,0.002935466667016347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,8192,0.03775360186894734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,7168,0.033111466964085894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,12288,0.05605760018030802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,10240,0.04556373357772827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,16384,0.07210026582082113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,3072,0.016054399808247886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,3584,0.018707199891408285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,5120,0.024805333216985068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,4096,0.020845866203308104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,6144,0.029021867116292316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,2560,0.013795199990272521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,512,0.003806933263937632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,2048,0.012121599912643433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,1536,0.009773866335550944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,1024,0.007412266731262207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,768,0.006318933268388112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,65536,0.30625813802083335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,256,0.003190399954716364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,32,0.0028970666229724885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,128,0.0030826665461063385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,64,0.0028704000016053517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,12288,0.04455893437067668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,7168,0.026949334144592284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,8192,0.03065920074780782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,10240,0.037861335277557376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,16384,0.06712106863657633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,3072,0.013722667098045349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,3584,0.015424000223477683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,4096,0.017143466075261436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,6144,0.02410986622174581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,5120,0.020363734165827433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,2560,0.011839999755223592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,2048,0.010128000378608703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,1024,0.006716800232728322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,1536,0.008549333612124125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,512,0.0035573333501815797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,768,0.005460266768932342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,65536,0.2670826594034831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,256,0.00317546675602595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,32,0.0029077333708604175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,64,0.0028042666614055633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,128,0.0030645333230495454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,7168,0.024675200382868447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,8192,0.02730026642481486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,10240,0.03282026648521423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,16384,0.05662933190663656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,12288,0.038496001561482744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,4096,0.016170666615168253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,6144,0.022055466969807945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,3072,0.011573333541552227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,3584,0.012961066762606301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,5120,0.01945706605911255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,65536,0.22589972813924156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,2560,0.010140800476074218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,2048,0.008714666962623597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,1536,0.007468800246715546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,1024,0.005898666878541311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,512,0.0035232000052928926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,256,0.0032479998966058097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,768,0.00415040006240209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,128,0.002993066608905792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,64,0.0027637332677841187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,32,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,10240,0.029666133721669513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,7168,0.022545067469278972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,8192,0.024678399165471397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,12288,0.0341973344484965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,16384,0.04396799802780151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,3072,0.010597333312034607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,4096,0.012928000092506409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,65536,0.1792736053466797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,3584,0.011799466609954835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,5120,0.017640533049901326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,6144,0.019997866948445638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,2560,0.009406933188438415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,1536,0.007607466479142506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,2048,0.008168533444404602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,1024,0.004795733094215393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,256,0.0031658666829268134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,768,0.0037952000896135964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,512,0.003522133330504099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,64,0.0027456000447273255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,128,0.0029578665892283124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,32,0.0028575999041398365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,12288,0.030039467414220172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,16384,0.039153067270914714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,10240,0.026572799682617186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,8192,0.022356265783309938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,7168,0.020153599977493285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,65536,0.1566538651784261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,6144,0.017851734161376955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,5120,0.015875200430552162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,4096,0.014197333653767904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,2048,0.0076000000039736434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,3072,0.01188053290049235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,3584,0.01311360001564026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,2560,0.009675733248392741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,768,0.0038730666041374207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,1024,0.004320000112056732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,1536,0.006670933465162914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,512,0.0034986667335033415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,128,0.0029279999434947968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,256,0.0031829332311948144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,64,0.0027818667391935987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,32,0.0027882667879263563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,8192,0.019364267587661743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,65536,0.1367743968963623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,12288,0.02645866672197978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,10240,0.02266133427619934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,7168,0.01767359972000122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,16384,0.03293226758639018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,6144,0.015763200322786965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,5120,0.015575466553370157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,3072,0.010629333058993022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,4096,0.01229866643746694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,3584,0.011270399888356526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,2048,0.0070698668559392285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,2560,0.00810346653064092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,1536,0.005736533304055532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,1024,0.004109866668780645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,768,0.0037834666669368743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,256,0.0031818665564060213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,64,0.0027978666126728057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,512,0.00344106654326121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,128,0.0029397333661715193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,32,0.0027797333896160126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,12288,0.02196586728096008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,65536,0.11588906447092692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,7168,0.014802133043607077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,16384,0.027133866151173906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,10240,0.019296000401178993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,8192,0.016008533040682473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,6144,0.013268267114957174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,5120,0.013095466295878091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,4096,0.010841600100199382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,3584,0.01014293332894643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,2560,0.007499733567237854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,3072,0.009489066402117411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,2048,0.006528000036875407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,1536,0.004757333298524221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,1024,0.004025600105524063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,768,0.003703466554482778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,128,0.0029098667204380036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,512,0.003387733300526937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,256,0.0031167998909950255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,32,0.0027189334233601887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,64,0.0027445333699385325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,65536,0.09213866392771403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,16384,0.02193066676457723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,10240,0.015876266360282897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,8192,0.013806933164596557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,7168,0.012852266430854797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,12288,0.01810773412386576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,6144,0.012658133109410604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,5120,0.01055680016676585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,4096,0.009646933277448018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,1536,0.0047189335028330484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,3584,0.009109333157539368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,3072,0.0086218665043513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,2048,0.005793066819508871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,2560,0.006903466582298279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,768,0.0036917333801587426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,1024,0.004036266605059306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,512,0.0033439998825391137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,65536,0.07133333683013916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,128,0.002902399996916453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,256,0.002994133283694585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,32,0.0028010666370391845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,64,0.002721066772937775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,7168,0.010345600048700969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,12288,0.01381119986375173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,10240,0.012585600217183432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,8192,0.010968533158302308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,6144,0.009322667121887207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,16384,0.016695467631022136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,5120,0.008931199709574383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,4096,0.00819946676492691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,3584,0.007715199887752533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,3072,0.006896000107129414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,1536,0.00470719983180364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,2048,0.005371733506520589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,2560,0.006062933305899302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,65536,0.04864533344904582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,256,0.0031456001102924346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,1024,0.004026666780312856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,768,0.0036618667344252265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,512,0.00335359995563825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,128,0.002844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,64,0.0027615999182065325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,16384,0.013742933670679728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,32,0.0027850667635599775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,12288,0.011918933192888895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,10240,0.010983467102050781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,8192,0.0099263995885849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,7168,0.00906880001227061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,6144,0.008408533533414204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,4096,0.006635733445485433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,5120,0.007906133433183034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,3584,0.007029333213965099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,3072,0.006680533289909363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,2048,0.005355733136336008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,65536,0.03835626840591431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,2560,0.0060479998588562015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,1536,0.004679466784000397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,256,0.0031360000371932983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,768,0.0036799999574820197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,128,0.0029045333464940387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,512,0.003339733431736628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,1024,0.004019200056791306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,64,0.002765866617361705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,12288,0.00957973301410675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,32,0.0027562665442625684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,10240,0.009180800120035807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,16384,0.011033599575360615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,8192,0.008338133494059246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,6144,0.006631466746330261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,7168,0.0071829333901405334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,5120,0.00697386662165324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,65536,0.028060799837112425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,4096,0.006379733482996623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,2560,0.00602346658706665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,3584,0.0069013332327206925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,3072,0.0065290664633115125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,1536,0.004692266881465912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,2048,0.005362133185068766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,512,0.0033130665620168054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,768,0.0036533333361148832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,1024,0.004012800008058548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,256,0.0031189332405726117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,128,0.002914133419593175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,64,0.0027594665686289472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,12288,0.007091199855009715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,32,0.002754133443037669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,16384,0.008214400211970011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,10240,0.006664533416430156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,65536,0.016936532656351724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,6144,0.00636053333679835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,8192,0.0066453332702318835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,7168,0.006567466755708058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,4096,0.006331733365853627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,5120,0.0067775999506314594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,2560,0.005998933315277099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,3584,0.006849066913127899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,3072,0.006375466783841451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,1536,0.0046623999873797095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,2048,0.005328000088532766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,1024,0.004031999905904134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,512,0.0033717334270477297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,768,0.0036789332826932275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,128,0.0028768000503381092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,256,0.003005866706371307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,64,0.0027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,32,0.0027413333455721537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,16384,0.006437333424886067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,65536,0.012205866972605388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,10240,0.006387199958165486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,12288,0.006415999929110209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,8192,0.006364800035953522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,7168,0.006258133550484974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,5120,0.00645653357108434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,3584,0.006495999793211619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,6144,0.006217599908510844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,4096,0.0061152001221974695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,3072,0.00629013329744339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,2560,0.005730133255322774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,2048,0.005109333495299021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,1024,0.0038399999340375268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,1536,0.004498133560021719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,512,0.003205333401759466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,768,0.00352960005402565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,128,0.002833066632350286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,256,0.002930133293072383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,32,0.0027285332481066385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,64,0.00271573339899381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,65536,0.009428266684214275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,16384,0.006244266529877981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,7168,0.006133333345254262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,12288,0.006270933151245117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,10240,0.006354133288065593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,8192,0.006291200220584869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,6144,0.0060586666067441305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,5120,0.006492800017197926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,3584,0.006482133269309997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,4096,0.00600853314002355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,2560,0.00558186670144399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,3072,0.006159999966621399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,1536,0.004437333345413208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,2048,0.005016533533732096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,768,0.0034634667138258614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,1024,0.00378560001651446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,512,0.0031871999303499854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,256,0.002932266642649968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,128,0.002746666719516118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,64,0.0026389333109060925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,65536,0.00791786660750707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,16384,0.006252799928188324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,32,0.0027104000250498454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,12288,0.006277333199977875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,6144,0.005994666616121928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,10240,0.006312533219655354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,8192,0.006151466568311056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,7168,0.006123733520507812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,5120,0.006413866579532623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,4096,0.006007466713587443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,3584,0.006472533444563548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,3072,0.0060703997810681665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,1024,0.003828266759713491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,2560,0.005594666798909505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,1536,0.004423466821511587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,2048,0.005010133484999338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,512,0.003160533308982849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,768,0.0035071998834609987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,256,0.0029919999341169994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,128,0.002810666710138321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,64,0.0026389333109060925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,32,0.002643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,5120,0.2252415974934896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,6144,0.27024106979370116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,7168,0.314193058013916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,8192,0.36173652013142904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,4096,0.1834496021270752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,10240,0.4511637369791667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,3072,0.14135252634684245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,3584,0.1592352072397868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,2560,0.1359498659769694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,1024,0.05375039974848429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,512,0.025788799921671553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,768,0.043277867635091144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,12288,0.5418986638387044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,1536,0.08264319896697998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,128,0.008829866846402485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,256,0.014512000481287637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,64,0.006265600025653839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,2048,0.09562239646911622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,32,0.004401066899299621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,7168,0.0782965342203776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,8192,0.0892416000366211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,10240,0.11001493136088054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,12288,0.13237333297729492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,16384,0.7185813268025716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,16384,0.17407253583272297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,6144,0.06844800313313802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,5120,0.05720426638921102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,3584,0.04632746775945028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,4096,0.04623786608378093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,1024,0.013955199718475341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,3072,0.041900801658630374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,768,0.01123520036538442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,1536,0.01968533396720886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,2560,0.03529813289642334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,2048,0.024794665972391765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,512,0.008492799599965413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,128,0.0034506666163603462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,256,0.005514666438102722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,32,0.003115733216206233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,64,0.0031178665657838186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,7168,0.06113919814427694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,8192,0.06953492959340414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,10240,0.08349013328552246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,12288,0.10539093017578124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,16384,0.1370346705118815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,5120,0.04385600090026855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,6144,0.053608532746632895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,4096,0.035674667358398436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,3584,0.03273599942525228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,3072,0.03203199903170268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,2560,0.023883734146753946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,1024,0.011710932850837708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,2048,0.01978879968325297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,768,0.00972266693909963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,512,0.007523199915885926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,1536,0.015708800156911215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,256,0.003991466760635376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,128,0.0035349334279696143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,32,0.003253333270549774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,64,0.0032458665470282235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,8192,0.058429865042368564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,10240,0.0723861296971639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,12288,0.08606080214182535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,16384,0.11319786707560223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,7168,0.05127786795298258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,6144,0.04511679808298747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,5120,0.03805013497670491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,4096,0.032092799743016556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,3584,0.02735146681467692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,3072,0.02407360076904297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,2560,0.02040106654167175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,1024,0.010026666522026061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,1536,0.013337600231170654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,2048,0.01686933239301046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,768,0.008246399958928426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,512,0.0065311998128890995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,65536,0.7018965403238933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,256,0.0036202666660149893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,128,0.003180799881617228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,64,0.0029728000362714132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,32,0.0030250666042168934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,8192,0.045933866500854494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,65536,0.5243189175923665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,10240,0.05664746761322022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,12288,0.06739839712778727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,16384,0.08796160221099854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,7168,0.040611199537913006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,6144,0.04158506790796916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,5120,0.030128000179926555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,4096,0.024728532632191977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,3584,0.02210879921913147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,1024,0.008520533641179402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,1536,0.011192533373832702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,2048,0.013911466797192892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,3072,0.019579732418060304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,2560,0.016809600591659545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,768,0.0071946665644645694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,512,0.005790933469931285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,128,0.0032490665713946023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,256,0.0035466666022936503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,64,0.002946133414904277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,32,0.003035733352104823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,65536,0.42316160202026365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,8192,0.04366506735483806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,10240,0.053369601567586265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,12288,0.06287146806716919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,7168,0.04503039916356404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,16384,0.0808288017908732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,6144,0.0348906675974528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,5120,0.02951253255208333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,4096,0.022504534324010214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,3072,0.017777067422866822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,3584,0.020118399461110433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,2560,0.015382400155067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,2048,0.012863999605178833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,1536,0.010475732882817586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,1024,0.007976533472537994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,768,0.006743466854095459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,512,0.004307200014591217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,256,0.0034080001215140024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,64,0.002940800040960312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,128,0.0030901332696278887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,65536,0.3823093414306641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,32,0.002946133414904277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,12288,0.05494399865468343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,10240,0.04664746522903442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,8192,0.0434442679087321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,7168,0.033758934338887533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,16384,0.0714143991470337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,6144,0.028586665789286297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,3584,0.018678400913874307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,4096,0.020721065998077392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,5120,0.024932267268498738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,3072,0.016565333803494772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,2560,0.013664000233014426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,1024,0.007436800003051758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,512,0.003912533322970072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,2048,0.012088533242543538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,1536,0.009779199957847595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,768,0.006289066871007283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,128,0.0030570665995279947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,256,0.003356799980004629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,64,0.002906666696071625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,65536,0.30526612599690756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,32,0.002867199977238973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,7168,0.02709439992904663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,8192,0.03545279900232951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,12288,0.04430506626764934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,16384,0.057956266403198245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,10240,0.03713173468907674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,3584,0.015312000115712484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,3072,0.013552000125249227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,5120,0.020595200856526694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,4096,0.01698453426361084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,6144,0.02421226700146993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,2560,0.011890133221944172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,65536,0.2649717330932617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,2048,0.009962667028109233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,1024,0.006712533533573151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,1536,0.008577066659927367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,512,0.0036874666810035707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,768,0.0055285334587097164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,256,0.00335359995563825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,128,0.0030517332255840302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,64,0.0028778667251269023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,32,0.0028661333024501802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,12288,0.0379530668258667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,7168,0.024413865804672242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,8192,0.027012266715367633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,10240,0.03263466755549113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,16384,0.04851733446121216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,6144,0.021733333667119346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,3584,0.012937600413958231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,65536,0.22518720626831054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,3072,0.011570133765538533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,4096,0.016157866517702738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,5120,0.01923840045928955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,2560,0.010245333115259807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,768,0.003961600114901861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,1536,0.008160000046094257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,256,0.0032000000278155005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,512,0.003483733286460241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,2048,0.008798933029174805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,1024,0.005843199789524078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,32,0.0028223998844623564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,128,0.002976000060637792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,64,0.0026943999032179515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,12288,0.034340266386667886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,10240,0.029462399085362752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,7168,0.022331732511520385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,8192,0.024456532796223958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,16384,0.043649065494537356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,65536,0.17640533447265624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,4096,0.012939733266830445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,3584,0.011794132987658183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,2560,0.010450133681297302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,5120,0.017426133155822754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,6144,0.019740800062815346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,3072,0.01060693363348643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,1024,0.004914133250713349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,2048,0.008214400211970011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,1536,0.007037866612275441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,768,0.003833599885304769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,512,0.0035146666069825493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,256,0.003155199935038885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,128,0.0029919999341169994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,64,0.002765866617361705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,32,0.002773333340883255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,10240,0.025974400838216144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,7168,0.020055466890335084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,8192,0.0220853328704834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,12288,0.03041279911994934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,16384,0.03872533241907756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,65536,0.1558143933614095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,6144,0.017630932728449504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,4096,0.014409599701563516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,5120,0.01573973298072815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,3584,0.013058132926623025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,2048,0.007593599955240886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,3072,0.011839999755223592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,2560,0.009593600034713745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,1536,0.0066101332505544026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,1024,0.004216533402601878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,512,0.003469866762558619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,768,0.003788800040880839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,256,0.0031637333333492277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,128,0.002926933268706004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,64,0.002765866617361705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,32,0.0027583998938401537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,65536,0.1358293374379476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,7168,0.0176256000995636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,8192,0.019309866428375243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,10240,0.02271359960238139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,12288,0.02617599964141846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,6144,0.017479467391967773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,16384,0.03245760003725688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,2560,0.008134399851163227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,4096,0.012307199835777282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,5120,0.014052266875902811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,3584,0.011447466413180033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,3072,0.010668800274531046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,2048,0.0070816000302632645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,1536,0.005657599866390228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,1024,0.004106666644414266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,768,0.0037813333173592886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,64,0.0027882667879263563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,512,0.0034656000634034475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,256,0.003160533308982849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,128,0.0029205332199732465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,32,0.0027285332481066385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,65536,0.11537386576334636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,6144,0.014327466487884521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,7168,0.014722133676211039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,8192,0.016037333011627197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,12288,0.02172373334566752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,10240,0.01916266679763794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,16384,0.02696746587753296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,5120,0.012306132912635803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,4096,0.010808533430099488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,3072,0.009689600268999735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,3584,0.010270933310190838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,2560,0.007493333518505096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,2048,0.006551466882228851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,1536,0.004725333551565806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,1024,0.004018133382002512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,768,0.0036618667344252265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,512,0.0033471999069054925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,64,0.00276053324341774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,128,0.002937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,256,0.003050666550795237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,32,0.0026943999032179515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,65536,0.09153813521067301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,10240,0.015658666690190635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,8192,0.013612799843152366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,7168,0.012744533022244773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,12288,0.017755732933680216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,6144,0.012685867150624594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,5120,0.010506666700045268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,16384,0.022048000494639078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,3584,0.009097599983215332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,4096,0.009559466441472372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,2048,0.005412266651789347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,3072,0.008559999863306682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,2560,0.007076266904671986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,1536,0.004692266881465912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,512,0.003369600077470144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,1024,0.0040287998815377556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,256,0.0031285333136717476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,128,0.0028970666229724885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,768,0.0037248000502586366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,65536,0.07104960282643637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,64,0.0027402666707833606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,32,0.002784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,6144,0.009305600325266521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,12288,0.013771733641624451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,8192,0.010936533411343891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,10240,0.012422399719556172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,7168,0.01018346647421519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,16384,0.016489600141843162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,5120,0.009035733342170716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,4096,0.008151466647783916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,3584,0.00755626658598582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,3072,0.006899199883143107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,2560,0.006027733286221823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,2048,0.005368533233801523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,65536,0.04810133377710978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,1536,0.004725333551565806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,1024,0.0040224000811576845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,768,0.003673599908749262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,32,0.0031082667410373688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,128,0.0029098667204380036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,512,0.0033717334270477297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,256,0.003044266750415166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,64,0.002977066735426585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,12288,0.01171946624914805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,16384,0.01394773324330648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,10240,0.010807466506958009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,6144,0.008356266220410665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,8192,0.009864532947540283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,7168,0.009080533186594646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,5120,0.007850666840871174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,4096,0.006677333513895671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,3584,0.007090133428573608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,3072,0.0065301333864529925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,65536,0.03808426856994629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,2560,0.006035199761390686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,2048,0.005319466690222422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,1536,0.004654933512210846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,768,0.003638399889071783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,1024,0.004021333406368891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,512,0.003366400053103765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,32,0.002869333326816559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,256,0.00325546662012736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,128,0.0030805334448814393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,64,0.0028607999285062153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,10240,0.009165866176287334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,16384,0.011062399546305338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,8192,0.008022400240103405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,12288,0.009750399986902874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,7168,0.007203199962774913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,5120,0.006995200117429097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,6144,0.006650666892528534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,65536,0.02768106659253438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,4096,0.006345599889755249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,3584,0.006761600077152252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,2048,0.0054293334484100345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,2560,0.0061258668700853985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,3072,0.006477866570154827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,1536,0.004762666424115499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,1024,0.004145066688458125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,256,0.003138133386770884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,768,0.0038111999630928038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,512,0.00345920001467069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,16384,0.008072533210118612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,128,0.0029130667448043824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,64,0.0026954665780067446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,32,0.00276799996693929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,10240,0.006806399921576183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,12288,0.006859733164310456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,8192,0.006630399823188781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,65536,0.0168287992477417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,7168,0.006458666423956554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,6144,0.006386133531729381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,5120,0.0066101332505544026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,4096,0.006230400005976359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,3584,0.006738133231798808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,3072,0.006392533580462138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,2560,0.0060245335102081295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,2048,0.005383466680844625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,1536,0.004691199958324432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,1024,0.004005333284536997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,256,0.003107200066248576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,768,0.0036447999378045404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,512,0.0033333333830038703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,128,0.002898133297761281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,64,0.0027434666951497394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,16384,0.006460799773534138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,8192,0.006274133423964183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,12288,0.006248533229033152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,32,0.002716800073782603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,10240,0.006455466647942861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,65536,0.011830400427182515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,5120,0.0064629331231117245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,6144,0.006133333345254262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,7168,0.006297599772612255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,4096,0.006031999985376993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,3072,0.006263466676076253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,3584,0.006525866687297821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,2560,0.005686399837334951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,2048,0.0050911997755368555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,1536,0.004524800181388855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,1024,0.003861333429813385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,768,0.003640533238649368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,512,0.003324799984693527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,256,0.0030293333033720653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,64,0.0027776000400384264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,128,0.002856533229351044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,32,0.002757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,16384,0.006144000093142191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,12288,0.006129066646099091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,65536,0.009589333335558574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,8192,0.006275199850400289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,6144,0.006039466460545858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,10240,0.006306133170922597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,7168,0.006128000219662985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,5120,0.0063733334342638654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,4096,0.005992533266544342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,2560,0.005611733098824819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,3072,0.006131199995676676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,3584,0.006494933366775512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,2048,0.004980266590913137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,1536,0.00443200021982193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,1024,0.003740799923737844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,768,0.0034773332377274835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,512,0.003223466624816259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,256,0.0030069333811601003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,128,0.00290133332212766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,64,0.0026933332284291584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,12288,0.006260266900062561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,32,0.0027093333502610523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,65536,0.008228266735871632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,16384,0.00625600020090739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,10240,0.006323199967543285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,8192,0.0062943999965985615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,6144,0.006041599810123444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,7168,0.006110933423042297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,3072,0.006086400151252747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,4096,0.005976533393065134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,3584,0.006504533191521962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,5120,0.006377600133419037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,1024,0.0037621334195137024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,2560,0.005613866448402405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,2048,0.0050122668345769245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,768,0.0034773332377274835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,512,0.0031370667119820913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,1536,0.0044608001907666525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,256,0.0028543998797734577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,128,0.00279573326309522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,64,0.0027317332724730173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,32,0.002713600049416224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,5120,0.22456852595011392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,6144,0.2704575856526693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,7168,0.3137834548950195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,8192,0.3624256134033203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,4096,0.18237226804097492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,10240,0.4506357192993164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,3584,0.15904852549235027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,3072,0.141102933883667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,2560,0.13508052825927735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,768,0.0433077335357666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,2048,0.09530986944834391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,1024,0.05589226484298706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,1536,0.08248106638590494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,128,0.009948800007502239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,256,0.014477866888046264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,12288,0.539353624979655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,64,0.00624533345301946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,512,0.025542400280634564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,32,0.004356266558170318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,7168,0.07800426483154296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,8192,0.08986453215281168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,10240,0.10994880199432373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,12288,0.15691626866658528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,16384,0.7096149444580078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,16384,0.17410346666971843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,5120,0.057180798053741454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,4096,0.046217600504557296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,6144,0.06824853420257568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,3072,0.041915734608968094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,3584,0.04475413163503011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,2560,0.03542399803797404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,1536,0.019598933060963948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,1024,0.016126933693885803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,768,0.011269332965215047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,512,0.008373333017031352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,256,0.0054848000407218935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,2048,0.024830933411916098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,32,0.0032170665760835014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,128,0.0034826666116714476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,64,0.003081600119670232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,7168,0.059291732311248777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,8192,0.06757866541544597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,10240,0.08648746808369955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,12288,0.10482772986094158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,16384,0.13637973467508951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,6144,0.057767466704050696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,5120,0.04418026606241862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,4096,0.035916801293691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,3584,0.03168639938036601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,3072,0.029476267099380494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,1024,0.011618133385976155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,768,0.00955733358860016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,1536,0.01569493313630422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,2560,0.023908267418543495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,2048,0.019538132349650066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,256,0.004093866546948751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,512,0.007456000149250031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,128,0.003454933315515518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,64,0.003173333406448364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,32,0.003189333279927572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,8192,0.05729813178380331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,10240,0.0721290667851766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,12288,0.08618026574452718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,16384,0.10890773137410481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,7168,0.0508565346399943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,5120,0.03778773148854574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,6144,0.04513066609700521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,4096,0.03151146570841472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,65536,0.7013024012247722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,2560,0.020451200008392335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,3072,0.023890133698781332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,3584,0.027193599939346315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,1536,0.015235199530919393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,2048,0.01667733391125997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,768,0.00827519992987315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,256,0.00367253323396047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,1024,0.00995306670665741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,512,0.006597333153088887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,32,0.0030762667457262674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,128,0.0033333333830038703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,64,0.002948266764481862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,65536,0.5211018562316895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,8192,0.045747200647989913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,12288,0.06719786326090495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,10240,0.05682133436203003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,16384,0.08837973276774089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,5120,0.0300981342792511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,6144,0.035641598701477054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,4096,0.02477226654688517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,7168,0.04764053424199422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,3584,0.022094933191935222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,3072,0.019495467344919838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,1536,0.011239467064539592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,1024,0.008556800087292989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,2560,0.016713599363962807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,2048,0.013939199844996133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,768,0.007231999933719635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,512,0.00556160012880961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,128,0.0031701333820819853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,256,0.0035968000690142312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,32,0.0030261332790056865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,64,0.0029237332443396253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,65536,0.4417087872823079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,7168,0.04478079875310262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,8192,0.04347626765569051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,10240,0.053053867816925046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,12288,0.06268373330434164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,16384,0.08075839678446452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,6144,0.03436799844106038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,4096,0.022408533096313476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,5120,0.029443200429280596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,3584,0.019938133160273232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,2048,0.01286079982916514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,3072,0.017758933703104655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,2560,0.015273599823315939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,768,0.006770133475462596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,1536,0.010473600029945374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,1024,0.007948799928029378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,512,0.004175999760627746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,128,0.0032149332265059153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,65536,0.35350933074951174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,256,0.0034133332471052804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,64,0.0028522667785485585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,32,0.00288426677385966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,7168,0.033920001983642575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,10240,0.04782293240229289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,8192,0.03825386762619019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,12288,0.06499946514765421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,16384,0.07117760181427002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,5120,0.023836799462636313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,4096,0.020805333058039346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,6144,0.02954026659329732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,3584,0.01871253252029419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,2560,0.013739732901255288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,3072,0.01583146651585897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,768,0.00619946668545405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,2048,0.012103466192881267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,1536,0.009818666179974874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,1024,0.007419733206431071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,512,0.003917866696914038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,256,0.00329066663980484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,64,0.0029397333661715193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,128,0.003094399968783061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,32,0.002951466788848241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,65536,0.3047072092692057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,7168,0.03155946731567383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,8192,0.030635732412338256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,16384,0.05808959801991781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,10240,0.036899201075236004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,12288,0.04467946688334147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,4096,0.01700906753540039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,3072,0.013676800330479941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,5120,0.020523732900619505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,6144,0.024037333329518636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,3584,0.015188266833623251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,2560,0.011955199639002483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,2048,0.01016533374786377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,65536,0.26357973416646324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,512,0.00373333344856898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,1536,0.008523733417193095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,1024,0.006691200037797292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,768,0.005143466591835022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,64,0.002880000074704488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,256,0.0032287999987602235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,128,0.003018666555484136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,32,0.002784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,12288,0.038186665376027426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,7168,0.024370133876800537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,8192,0.026923733949661254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,10240,0.032501333951950075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,16384,0.048493866125742593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,6144,0.02188160022099813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,3584,0.012964266538619994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,65536,0.22535786628723145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,4096,0.016328533490498863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,5120,0.01909546653429667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,3072,0.011619200309117634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,2560,0.010154666503270467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,1024,0.005818666517734527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,2048,0.008899199962615966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,1536,0.00823466678460439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,768,0.003934933245182038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,256,0.0031839999059836066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,512,0.003572266548871994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,32,0.0028138667345046996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,128,0.002995199958483378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,64,0.002791466563940048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,10240,0.029193600018819172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,7168,0.022094933191935222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,16384,0.04320746660232544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,12288,0.03395306666692098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,8192,0.02426240046819051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,65536,0.17595626513163248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,4096,0.012946133812268576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,3072,0.010569600264231364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,3584,0.01188800036907196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,5120,0.01737066706021627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,6144,0.019500799973805747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,2048,0.008258133133252462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,2560,0.010478933652242024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,1536,0.007052800059318543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,256,0.003271466741959254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,1024,0.004902400076389313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,512,0.0035445332527160645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,768,0.003899733225504557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,128,0.002924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,32,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,64,0.002726399898529053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,8192,0.021643733978271483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,7168,0.01983893314997355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,12288,0.030291199684143066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,10240,0.02632533311843872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,65536,0.15537066459655763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,16384,0.03739413420359294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,5120,0.01560640037059784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,2560,0.008754133184750875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,6144,0.01750719944636027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,4096,0.014203733205795288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,3584,0.013069867094357809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,3072,0.012893866499265036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,1536,0.00661653329928716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,1024,0.004329599936803182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,2048,0.00766293356815974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,512,0.003521066655715307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,256,0.0031744000812371576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,768,0.0039007999002933502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,128,0.0029738667110602063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,64,0.002811733384927114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,32,0.002791466563940048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,8192,0.01927466591199239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,12288,0.026180267333984375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,10240,0.022629332542419434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,7168,0.017411200205485027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,16384,0.0328277329603831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,65536,0.1370954672495524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,6144,0.015686399737993875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,4096,0.012191999951998394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,5120,0.01402773360411326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,3584,0.011373866597811382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,3072,0.010627200206120808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,2560,0.008772266904513042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,2048,0.0070816000302632645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,1536,0.005814399818579356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,768,0.0038101332883040107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,512,0.003469866762558619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,1024,0.0041461333632469176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,256,0.0031498665610949195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,128,0.002959999938805898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,64,0.0028362666567166646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,32,0.0027978666126728057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,65536,0.11514773368835449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,8192,0.016080000003178916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,7168,0.014773333072662353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,10240,0.018961066007614137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,12288,0.02166293263435364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,16384,0.026884265740712482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,5120,0.01223360002040863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,6144,0.014389333128929139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,3584,0.010291199882825215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,4096,0.010942932963371278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,3072,0.009611733754475911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,2560,0.007422933479150136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,1024,0.004045866678158442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,2048,0.006532266736030579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,1536,0.004728533327579498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,768,0.003706666578849157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,128,0.0028778667251269023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,512,0.0034005333979924522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,256,0.003065599997838338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,64,0.002762666592995326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,32,0.002721066772937775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,65536,0.09103893438975016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,8192,0.013547733426094055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,10240,0.01595626672108968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,7168,0.012754133343696595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,6144,0.012759466965993246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,16384,0.021754666169484457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,12288,0.017812265952428182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,5120,0.010642133156458537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,3584,0.00909546713034312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,3072,0.008578133583068848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,4096,0.009637332955996196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,2560,0.007047466437021891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,1536,0.004729599754015604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,2048,0.005392000079154968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,65536,0.07071466445922851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,1024,0.004037333279848098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,768,0.0036490666369597114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,128,0.002951466788848241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,256,0.003155199935038885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,512,0.0033952000240484873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,64,0.002755200117826462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,12288,0.013736533125241599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,32,0.002754133443037669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,10240,0.012421333789825439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,8192,0.011026133100191753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,16384,0.016562133034070333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,7168,0.010180266698201497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,6144,0.009331199526786804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,4096,0.007933866480986278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,5120,0.008850133419036866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,3584,0.00769706666469574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,3072,0.0067445332805315655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,1536,0.0046847999095916745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,2048,0.005334400137265523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,2560,0.006051200131575266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,1024,0.004023466755946477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,65536,0.04809919993082683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,256,0.003078400095303853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,768,0.003692800054947535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,512,0.0033471999069054925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,128,0.0029290666182835894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,32,0.003047466774781545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,64,0.002996266633272171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,16384,0.013740799824396768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,8192,0.009768533706665038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,10240,0.010950400431950887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,12288,0.011890133221944172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,7168,0.009150933225949604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,6144,0.008343467116355896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,5120,0.007699200014273326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,4096,0.0067104001839955645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,65536,0.0380021333694458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,3584,0.007034666836261749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,3072,0.0064640000462532045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,2560,0.006007466713587443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,1536,0.004669866462548574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,1024,0.004006399959325791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,2048,0.005369600156943003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,512,0.0033151999115943907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,768,0.0036821333070596062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,256,0.003260799994071325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,128,0.0029472000896930695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,32,0.002882133424282074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,64,0.002882133424282074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,12288,0.009744000434875489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,16384,0.011069867014884948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,7168,0.007177599767843883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,6144,0.006724266707897187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,8192,0.008225066463152568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,10240,0.009296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,65536,0.02717439929644267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,4096,0.0062943999965985615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,5120,0.0067104001839955645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,3072,0.006358399987220764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,3584,0.006725333134333293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,2560,0.006128000219662985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,2048,0.005465599894523621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,512,0.0034389334420363107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,1536,0.004779733220736186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,1024,0.0041461333632469176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,256,0.0031615999837716425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,768,0.003786666691303253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,128,0.0028938665986061097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,64,0.002809600035349528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,32,0.0027935999135176343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,12288,0.0069461335738499955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,16384,0.008175999919573466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,10240,0.006580266853173573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,8192,0.006505600114663441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,65536,0.01703146696090698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,6144,0.006291200220584869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,5120,0.006611200173695882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,7168,0.006449066599210103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,4096,0.006292266647020976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,3584,0.0067775999506314594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,3072,0.00643093337615331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,1536,0.0046847999095916745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,2048,0.005341866612434387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,2560,0.006006399790445963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,768,0.0036501333117485045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,1024,0.00402453343073527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,512,0.0033439998825391137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,128,0.0029109333952267963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,256,0.0031040000418821974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,64,0.0027530667682488757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,32,0.002734933296839396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,65536,0.011970133582750956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,16384,0.006266666452089946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,12288,0.006217599908510844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,10240,0.006455466647942861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,8192,0.006291200220584869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,7168,0.00613973339398702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,5120,0.006425599753856659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,6144,0.006178133189678192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,3072,0.006223999957243601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,2048,0.005098666747411092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,4096,0.006089599927266439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,3584,0.006520533561706543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,2560,0.005701333284378052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,1536,0.0045045331120491024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,512,0.0033514666060606635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,768,0.0036703998843828833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,1024,0.0038165333370367683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,256,0.003045333425203959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,128,0.0028543998797734577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,64,0.002699733277161916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,32,0.0027306665976842242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,65536,0.00909546713034312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,16384,0.006182399888833364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,12288,0.006201600035031637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,8192,0.006217599908510844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,7168,0.0061034664511680605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,10240,0.006379733482996623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,5120,0.00639466643333435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,6144,0.006045866509278615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,4096,0.006027733286221823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,3072,0.00617386649052302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,2560,0.005569066603978475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,3584,0.006443733473618825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,1536,0.004410666724046072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,2048,0.005016533533732096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,768,0.0034613333642482757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,1024,0.003773866593837738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,256,0.003052799900372823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,512,0.0031199999153614043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,65536,0.008222933113574981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,64,0.0027669332921504974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,128,0.0028223998844623564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,16384,0.006324266890684764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,32,0.0027221334477265675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,8192,0.006205866734186808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,7168,0.006100266675154368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,12288,0.0061706667145093284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,10240,0.0063360000650088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,5120,0.006401066482067108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,6144,0.006046933432420095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,4096,0.006017066538333893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,3584,0.006470400094985962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,3072,0.006060799956321717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,2560,0.00553706685702006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,2048,0.004996266464392344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,1536,0.004436266422271728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,768,0.003489066660404205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,1024,0.0037205333511034647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,512,0.003256533294916153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,256,0.0029365333418051405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,128,0.0027637332677841187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,64,0.002696533252795537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,32,0.0026346666117509207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,5120,0.22435733477274575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,6144,0.26960748036702475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,7168,0.31375039418538414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,8192,0.36091413497924807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,10240,0.4506666819254558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,4096,0.18289492925008138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,3584,0.15860053698221843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,3072,0.14064000447591146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,2560,0.1356554667154948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,1536,0.0784661372502645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,512,0.02539520064989726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,2048,0.10760107040405273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,1024,0.05581119855244955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,768,0.043060266971588136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,12288,0.5440970738728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,128,0.008867200215657551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,64,0.0067114666104316715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,256,0.014386133352915446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,32,0.004454400142033895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,7168,0.0781045357386271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,8192,0.08850239912668864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,10240,0.10987199942270916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,12288,0.13226666450500488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,16384,0.7130101521809895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,4096,0.045949868361155194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,5120,0.05693013270696005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,16384,0.207861328125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,3584,0.047654398282368976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,6144,0.06821013291676839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,3072,0.041843199729919435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,768,0.012817066907882691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,1024,0.01393280029296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,512,0.008400000135103862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,2048,0.025044266382853193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,1536,0.022618667284647623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,128,0.0034613333642482757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,2560,0.031593600908915206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,256,0.005729066828886667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,64,0.0031040000418821974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,32,0.0030847998956839246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,8192,0.06998079617818197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,7168,0.06160533428192139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,10240,0.0844533363978068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,12288,0.1042954683303833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,16384,0.1364415963490804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,3584,0.03284800052642822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,5120,0.04425813357035319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,4096,0.03948160012563069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,6144,0.053428268432617186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,1024,0.011534933249155681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,3072,0.032237867514292404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,1536,0.01572480003039042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,2560,0.024898133675257363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,2048,0.01988160014152527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,128,0.0034741332133611047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,512,0.007419733206431071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,768,0.009589333335558574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,256,0.004259199897448222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,64,0.003188266605138779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,32,0.0032000000278155005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,8192,0.05817919969558716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,10240,0.07207252979278564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,12288,0.08603200117746988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,16384,0.11308586597442627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,7168,0.051267198721567785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,6144,0.04515093167622884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,5120,0.03774506648381551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,4096,0.03061760067939758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,3584,0.02723413308461507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,1536,0.013337600231170654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,3072,0.02402879993120829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,2560,0.02022506594657898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,1024,0.00993386705716451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,2048,0.01686506668726603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,768,0.00830080012480418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,512,0.006595199803511302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,256,0.0036501333117485045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,65536,0.6995253245035807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,64,0.0029045333464940387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,128,0.003311999887228012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,32,0.0029685333371162414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,8192,0.04573866526285807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,65536,0.5290453275044759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,10240,0.056415998935699464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,12288,0.0673845370610555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,16384,0.10417280197143555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,7168,0.04050346612930298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,5120,0.03012160062789917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,4096,0.024593067169189454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,6144,0.0421674648920695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,3584,0.0219541331132253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,1536,0.011251200238863628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,2048,0.013882666826248169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,1024,0.00844693382581075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,2560,0.016683733463287352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,3072,0.019368533293406168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,768,0.007191466788450877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,256,0.0034933333595593774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,512,0.005726933479309082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,64,0.002883200099070867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,128,0.0031317333380381264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,65536,0.44167038599650066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,32,0.0029493334392706556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,8192,0.05009706815083822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,10240,0.05262080033620199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,7168,0.03857386509577433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,12288,0.06250666777292888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,16384,0.0805023988087972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,6144,0.0341536005338033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,4096,0.022346667448679605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,3584,0.01983893314997355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,5120,0.029157332579294842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,3072,0.017744000752766928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,2560,0.01527253290017446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,65536,0.35306558609008787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,1536,0.01050453285376231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,1024,0.007952000200748443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,2048,0.012827733159065246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,768,0.00678719977537791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,512,0.004301866888999939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,256,0.0034805332620938623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,128,0.00311253344019254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,64,0.002935466667016347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,32,0.002921599894762039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,7168,0.03351680040359497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,12288,0.05452160040537516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,8192,0.038441598415374756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,10240,0.05013120174407959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,16384,0.0706666628519694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,5120,0.024024534225463866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,3584,0.018386133511861167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,4096,0.020872533321380615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,3072,0.015874133507410685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,6144,0.02942933241526286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,1024,0.007249066730340321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,2560,0.0135861337184906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,2048,0.012026666601498922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,1536,0.009835732976595561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,768,0.006250666578610738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,512,0.003992533435424169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,256,0.003294933338960012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,128,0.0031114667654037476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,65536,0.30464213689168296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,64,0.0028575999041398365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,32,0.0028597332537174227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,7168,0.026898133754730224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,10240,0.04405866861343384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,8192,0.030687999725341798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,12288,0.04376320044199626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,16384,0.05802666743596395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,3072,0.013622400164604188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,3584,0.015280000368754067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,4096,0.01702186663945516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,5120,0.020526933670043945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,2560,0.011823999881744384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,6144,0.023758933941523234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,65536,0.2643967946370443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,768,0.00580266664425532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,2048,0.010120532910029094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,1024,0.006626133124033611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,1536,0.008513066172599792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,512,0.003787733366092046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,128,0.003066666672627131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,256,0.003230933348337809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,64,0.0028351999819278715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,32,0.0028864001234372456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,12288,0.03770560026168823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,10240,0.032228267192840575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,8192,0.026947200298309326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,7168,0.024280534187952677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,16384,0.0483456015586853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,6144,0.021785600980122884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,3584,0.01279146671295166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,5120,0.01900586684544881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,4096,0.016174933314323424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,65536,0.2250688076019287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,2560,0.010260267059008281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,3072,0.011654399832089742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,2048,0.008970666925112407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,512,0.0035743998984495797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,1536,0.007604266703128815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,1024,0.005880533158779145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,768,0.004011733333269755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,256,0.0031850665807724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,128,0.002995199958483378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,64,0.00288426677385966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,32,0.0027690666417280836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,8192,0.024246400594711302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,10240,0.029041065772374468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,7168,0.022029866774876915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,12288,0.03371413151423137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,16384,0.043137065569559735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,65536,0.1761397361755371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,4096,0.012930132945378623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,3072,0.010628267129262289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,6144,0.019409066438674925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,3584,0.011866666873296102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,5120,0.017262933651606242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,2560,0.010570666193962098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,1536,0.0070613334576288865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,2048,0.008343467116355896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,1024,0.004870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,768,0.003953066716591517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,512,0.003570133447647095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,128,0.0029909332593282064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,256,0.0031669333577156065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,32,0.0028575999041398365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,64,0.0027146667242050173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,8192,0.021524266401926676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,7168,0.01996586720148722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,12288,0.029574400186538695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,10240,0.02613439957300822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,16384,0.03854613304138184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,65536,0.15434560775756836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,5120,0.015640532970428465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,4096,0.013778133193651834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,6144,0.017496534188588462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,3072,0.01188800036907196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,3584,0.013092266519864401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,2560,0.00970240036646525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,1536,0.006587733328342438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,2048,0.007670400043328603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,1024,0.004353066782156626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,768,0.003918933371702829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,512,0.003575466573238373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,128,0.00297173336148262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,256,0.0031306666632493338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,64,0.0026954665780067446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,32,0.002846933404604594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,7168,0.017410133282343546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,8192,0.019131733973821004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,10240,0.022402133544286093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,12288,0.026027733087539674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,65536,0.13679572741190593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,16384,0.03265920082728068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,6144,0.015621333320935567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,5120,0.01397119959195455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,3584,0.011413333813349406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,4096,0.012173866232236225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,3072,0.01141866644223531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,2560,0.008136533200740814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,2048,0.00709440012772878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,1536,0.0058335999647776285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,1024,0.004203733305136363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,768,0.0038176000118255613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,128,0.002958933264017105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,256,0.0032085334261258446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,512,0.0035264000296592714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,32,0.0028586665789286296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,64,0.0026954665780067446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,65536,0.11505173047383625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,7168,0.014709333578745524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,8192,0.016129066546758018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,16384,0.02669653296470642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,12288,0.021707733472188316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,10240,0.01882986625035604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,5120,0.012250666817029316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,6144,0.014633599917093912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,4096,0.010851200421651204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,3584,0.010115200281143188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,3072,0.009604266285896302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,2048,0.006485333542029063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,2560,0.007493333518505096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,1536,0.004839466512203216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,1024,0.004038399954636892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,768,0.003697066754102707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,256,0.0030005333324273427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,32,0.002780800064404805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,512,0.0033791999022165934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,64,0.002696533252795537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,128,0.002850133428970973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,65536,0.09081493218739828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,8192,0.01362559994061788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,12288,0.017655466000239053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,10240,0.015864533185958863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,6144,0.011779200037320454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,7168,0.012661332885424295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,16384,0.021819732586542764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,4096,0.009458133578300476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,5120,0.010588799913724262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,3584,0.009078400333722432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,3072,0.008508800466855367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,2560,0.006838400165239971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,2048,0.005358933409055074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,1536,0.004702933132648468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,65536,0.07053866386413574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,1024,0.004029866556326548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,64,0.0027935999135176343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,512,0.0033717334270477297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,768,0.003722666700681051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,128,0.002755200117826462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,256,0.0029845332105954488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,32,0.002810666710138321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,12288,0.013716266552607218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,8192,0.010937600334485372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,10240,0.01243839959303538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,6144,0.009367466966311137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,7168,0.010194133718808491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,16384,0.01660053332646688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,3072,0.006776533524195353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,4096,0.008021333316961924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,5120,0.008889599641164144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,2048,0.005345066885153452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,3584,0.00763733337322871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,2560,0.006031999985376993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,65536,0.04798719882965088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,1536,0.004701866706212362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,1024,0.004046933352947235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,768,0.0036874666810035707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,512,0.003403733422358831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,256,0.003091199944416682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,128,0.0028789333999156954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,64,0.002946133414904277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,32,0.0030613332986831666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,16384,0.013875200351079305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,12288,0.011750400066375732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,6144,0.008426666259765625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,10240,0.010964266459147136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,7168,0.009027199943860371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,8192,0.009799466530481974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,5120,0.007950933277606964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,4096,0.0066336000959078475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,3584,0.007074133555094402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,65536,0.03774506648381551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,2560,0.006027733286221823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,3072,0.006545066833496094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,2048,0.005353599786758423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,768,0.0036501333117485045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,1536,0.004724266628424326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,1024,0.004002133260170618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,512,0.003335466732581457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,256,0.003180799881617228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,32,0.0028661333024501802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,128,0.003047466774781545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,64,0.0028277332584063213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,16384,0.01104853351910909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,12288,0.009755733609199523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,10240,0.009111467003822326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,8192,0.008312533299128216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,6144,0.006814933319886525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,7168,0.0072053333123524976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,65536,0.027156267563501996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,5120,0.0069365332523981735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,3072,0.006439466774463653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,4096,0.006379733482996623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,3584,0.0068693334857622785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,1024,0.004107733319203059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,768,0.003772799919048945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,2560,0.006109866499900818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,1536,0.004785066843032837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,2048,0.005423999826113383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,512,0.0034506666163603462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,256,0.003050666550795237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,128,0.0029728000362714132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,64,0.0028138667345046996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,32,0.002808533360560735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,12288,0.006964266796906789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,16384,0.007797333101431529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,10240,0.0066997334361076355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,7168,0.006405333181222279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,65536,0.016835200786590575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,8192,0.0065087998906771345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,5120,0.006692266464233399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,6144,0.006251733501752217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,3584,0.0067893331249554946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,3072,0.006346666812896728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,4096,0.006298666695753734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,2560,0.006037333110968272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,2048,0.005355733136336008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,512,0.0034474665919939675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,1536,0.004705066482226053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,768,0.003655466685692469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,1024,0.004006399959325791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,64,0.0027285332481066385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,256,0.0032330666979153953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,128,0.002850133428970973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,32,0.0027413333455721537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,10240,0.006454400221506755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,65536,0.011834667126337687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,16384,0.006260266900062561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,8192,0.006455466647942861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,12288,0.006253866851329804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,7168,0.006263466676076253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,6144,0.006087466577688853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,5120,0.0064640000462532045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,4096,0.006101333101590474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,3584,0.0064640000462532045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,3072,0.0062613333264986675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,2048,0.005120000243186951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,2560,0.005650133391221364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,1024,0.003864533454179764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,1536,0.0044832001129786175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,768,0.0036330667634805044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,512,0.003305600086847941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,256,0.00306986669699351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,32,0.002746666719516118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,128,0.002863999952872594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,64,0.0027274665733178455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,65536,0.009178666273752849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,16384,0.006209066510200501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,12288,0.0061034664511680605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,8192,0.006242133180300395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,10240,0.006389333307743073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,7168,0.006131199995676676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,6144,0.006087466577688853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,5120,0.0064074665307998655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,3072,0.006146133442719777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,4096,0.006028800209363302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,3584,0.006488533318042755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,2048,0.005013333261013031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,1024,0.0037845333417256674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,2560,0.005594666798909505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,1536,0.004403199752171834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,768,0.0034133332471052804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,512,0.0031413334111372627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,256,0.0030346666773160298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,64,0.0027221334477265675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,128,0.0028607999285062153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,16384,0.006151466568311056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,32,0.0027295999228954316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,65536,0.007625600198904674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,12288,0.006214400132497152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,10240,0.006311466793219249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,6144,0.005975466469923655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,8192,0.006239999830722809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,7168,0.006126933296521505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,5120,0.006419200201829274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,3584,0.006454400221506755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,4096,0.005902933577696482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,3072,0.0061141331990559895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,2560,0.00561706672112147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,2048,0.005019733309745788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,1536,0.004442666471004486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,1024,0.0037930667400360107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,768,0.0034186666210492453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,256,0.0029205332199732465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,128,0.002808533360560735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,512,0.0031328000128269195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,64,0.0027232001225153605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,32,0.0026975999275843303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,5120,3.858075714111328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,6144,4.760126749674479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,7168,5.432294209798178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,8192,6.451955159505208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,4096,3.221033477783203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,3584,2.754741414388021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,10240,7.90427754720052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,3072,2.387269337972005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,2560,2.079270426432292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,1024,0.9199040095011393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,512,0.5942314783732097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,1536,1.2636778513590494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,12288,9.605373128255208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,768,0.7684842427571614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,256,0.4599434534708659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,128,0.42632427215576174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,2048,1.592123794555664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,64,0.4145407994588216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,32,0.4083104133605957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,16384,12.728799438476562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,7168,1.4921685536702474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,8192,1.708346684773763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,10240,2.110585657755534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,12288,2.599792989095052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,6144,1.3102036794026692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,4096,0.8761856079101562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,5120,1.0806954701741538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,3584,0.768069330851237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,1536,0.3667680104573568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,2560,0.5696661631266277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,2048,0.46699838638305663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,16384,3.369456990559896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,3072,0.6820202509562174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,1024,0.2668053309122721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,64,0.1210741360982259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,256,0.1346560001373291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,768,0.22106560071309408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,128,0.12440426349639892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,32,0.11857813199361164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,512,0.16597654024759928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,7168,1.1627925872802733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,8192,1.3047072092692056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,10240,1.6459434509277344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,12288,1.9485535939534504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,5120,0.8346282958984375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,6144,0.9921770731608073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,4096,0.6692874908447266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,16384,2.6166175842285155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,3584,0.5931957244873047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,1536,0.2860149383544922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,2048,0.36014614105224607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,2560,0.4406901359558105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,1024,0.20822827021280924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,3072,0.5125919977823894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,768,0.1734997272491455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,256,0.10812479654947918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,128,0.09750186602274577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,512,0.13282132943471273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,64,0.09623040358225504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,32,0.09415146509806314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,8192,1.1513888041178384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,10240,1.416653823852539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,12288,1.70839360555013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,16384,2.273062388102214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,7168,1.0178762435913087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,6144,0.8694186528523764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,5120,0.7295509338378906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,3584,0.527509339650472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,3072,0.45222613016764324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,4096,0.5960693359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,2560,0.38600212732950845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,2048,0.32099412282307943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,768,0.15691626866658528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,1024,0.18841387430826823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,1536,0.2520778656005859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,512,0.12536959648132323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,256,0.09401919841766357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,128,0.08420373598734537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,64,0.08264213403066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,32,0.08144853115081788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,65536,14.013346354166666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,10240,1.2321119944254557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,12288,1.4438891092936197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,8192,0.9624554951985678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,16384,1.94179204305013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,7168,0.8507882436116537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,6144,0.725277837117513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,5120,0.6117408116658528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,4096,0.48167572021484373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,65536,10.859552001953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,3072,0.3635498682657877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,3584,0.42543360392252605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,1536,0.21954453786214195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,2560,0.3062591870625814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,2048,0.2517493406931559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,512,0.10068906943003338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,128,0.07117226918538412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,1024,0.15658666292826334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,768,0.12964800198872883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,256,0.07970560391743978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,64,0.0704746643702189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,32,0.07002240022023519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,8192,0.8795605341593424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,10240,1.1133055369059244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,12288,1.304977035522461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,65536,9.407249959309896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,16384,1.7413888295491535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,7168,0.7701290766398112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,6144,0.6754037221272786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,5120,0.5573749542236328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,4096,0.4550901412963867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,3072,0.34854612350463865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,3584,0.40343360900878905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,2560,0.3015210787455241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,2048,0.2472853342692057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,1536,0.19660372734069825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,512,0.09170239766438802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,768,0.11995946566263835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,1024,0.1435167948404948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,128,0.06508266528447469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,64,0.06411733229955038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,256,0.07288533051808675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,32,0.06363306840260824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,8192,0.8035722732543945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,10240,0.9901407877604166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,12288,1.205117925008138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,65536,7.857835896809895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,16384,1.5707360585530599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,7168,0.7061098734537761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,6144,0.6071786880493164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,5120,0.5200640042622884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,4096,0.41738026936848954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,3584,0.37144320805867515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,2048,0.2297898610432943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,3072,0.3211445490519206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,2560,0.27433811823527016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,256,0.06606186628341675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,512,0.08278400103251139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,768,0.10884586970011394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,1536,0.18016533851623534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,1024,0.13160746892293293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,128,0.05792426665623983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,32,0.05768746534983317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,64,0.058456532160441076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,8192,0.7034144083658854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,10240,0.8848960240681967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,12288,1.041258684794108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,16384,1.3916053771972656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,7168,0.6187135696411132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,6144,0.5401354471842448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,5120,0.4554613431294759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,65536,7.030292256673176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,4096,0.3718933423360189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,3584,0.3267669359842936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,2048,0.20356906255086266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,3072,0.28642028172810874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,2560,0.2453312079111735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,256,0.0596127986907959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,512,0.07875626881917318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,768,0.09900373617808024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,1024,0.11674559911092122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,1536,0.1608736038208008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,64,0.05161386728286743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,128,0.051787734031677246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,32,0.051950931549072266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,8192,0.6115552266438802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,10240,0.7595530827840169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,12288,0.9150847752888998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,16384,1.2083263397216797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,7168,0.5378442764282226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,65536,6.38132069905599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,6144,0.46701440811157224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,5120,0.3960298538208008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,2048,0.17682560284932453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,3072,0.2512128035227458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,2560,0.21415786743164061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,3584,0.28492800394694007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,4096,0.3248832066853841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,256,0.05261973142623901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,1536,0.1386474609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,512,0.0670261303583781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,1024,0.10307839711507161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,768,0.08538453578948975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,64,0.045501867930094406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,32,0.04594026803970337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,128,0.04565226634343465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,8192,0.5711072285970051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,10240,0.7030645370483398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,12288,0.8487744013468423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,16384,1.1144213358561197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,65536,5.530024719238281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,7168,0.5023370742797851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,6144,0.4333205223083496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,5120,0.3689397176106771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,3072,0.2344810644785563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,3584,0.26732587814331055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,2048,0.16578879356384277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,2560,0.19857279459635419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,4096,0.302237860361735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,768,0.07968746821085612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,256,0.049081599712371825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,1024,0.09604053497314453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,512,0.06295146544774374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,1536,0.1298858642578125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,128,0.04402666489283244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,64,0.04241919914881388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,32,0.042238934834798174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,8192,0.5248096148173015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,10240,0.6493130366007487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,12288,0.7795989354451497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,16384,1.0281973520914713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,7168,0.4626261393229167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,65536,4.868837483723959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,6144,0.40224320093790694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,5120,0.34046719868977865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,3072,0.2182645320892334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,2560,0.1845311959584554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,3584,0.24797546068827309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,4096,0.2784149487813314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,2048,0.15332800547281902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,256,0.045569066206614176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,512,0.058888534704844155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,768,0.07459093729654948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,1536,0.12161920070648194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,1024,0.08897279898325602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,128,0.03959999879201253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,64,0.03984533150990804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,32,0.039027198155721025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,8192,0.4821632067362468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,10240,0.5968810399373372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,12288,0.7154495875040691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,65536,4.4762420654296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,16384,0.9386058807373047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,7168,0.4252778689066569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,6144,0.36948906580607094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,5120,0.31452693939208987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,2048,0.14202772776285807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,3072,0.19989013671875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,3584,0.22862186431884765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,2560,0.171395206451416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,4096,0.2586954593658447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,256,0.041621331373850504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,512,0.05552213191986084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,1024,0.08262826601664225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,768,0.06927146911621093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,1536,0.11173120339711506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,128,0.035895466804504395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,32,0.0345141331354777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,64,0.03495253324508667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,8192,0.44130986531575517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,10240,0.5431018829345703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,12288,0.6502911885579427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,16384,0.8596373240152995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,65536,4.050615437825521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,7168,0.3894463857014974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,6144,0.33752320607503256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,5120,0.2881919860839844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,4096,0.23444906870524088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,3072,0.18213119506835937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,3584,0.20931199391682945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,2560,0.15561493237813312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,2048,0.13110400040944417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,1536,0.10208319822947184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,768,0.06317866643269857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,1024,0.07537173430124919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,256,0.03842133283615112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,512,0.05016640027364096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,128,0.03232000072797139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,64,0.03142613371213277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,32,0.030856533845265703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,8192,0.39809707005818684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,10240,0.4918794631958008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,12288,0.5841077168782551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,16384,0.7688128153483074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,65536,3.738099161783854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,7168,0.3512085278828939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,6144,0.3060682614644369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,5120,0.2581226666768392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,3584,0.18989760080973309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,3072,0.1650325298309326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,4096,0.21024319330851235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,2048,0.11760533650716146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,2560,0.14255146980285643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,256,0.03327999909718831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,512,0.04500373204549153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,768,0.05707840124766032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,1024,0.06848533153533935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,1536,0.09467626412709554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,64,0.0279423991839091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,128,0.0291103998819987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,32,0.02818666696548462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,8192,0.35141226450602214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,10240,0.43755518595377607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,12288,0.5198346773783367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,65536,3.3904266357421875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,16384,0.6907722473144531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,7168,0.3101792017618815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,6144,0.2690944035847982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,5120,0.22642240524291993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,3584,0.16680533091227215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,3072,0.1452426592508952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,2560,0.12611839771270753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,4096,0.18667200406392415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,2048,0.10433066685994466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,256,0.026314665873845417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,768,0.046690134207407634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,512,0.035259731610616046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,1024,0.057630932331085204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,1536,0.08228586514790853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,128,0.022636799017588298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,64,0.022026666005452476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,32,0.02160960038503011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,8192,0.33045654296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,10240,0.4110431989034017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,12288,0.4884309450785319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,65536,3.0427894592285156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,16384,0.6490496317545573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,7168,0.2921056111653646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,6144,0.25382827123006185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,5120,0.2146549383799235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,4096,0.17692160606384277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,3584,0.1572853406270345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,2048,0.0978762706120809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,3072,0.1377386728922526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,2560,0.11867733001708984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,256,0.02480319937070211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,768,0.04081813494364421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,512,0.03175573348999024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,1024,0.05239466826121012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,1536,0.07642239729563395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,128,0.02104533314704895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,64,0.02055466572443644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,32,0.0201365331808726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,8192,0.3090751965840658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,10240,0.38153279622395836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,12288,0.4566656112670898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,65536,2.7139211018880207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,16384,0.6060138702392578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,7168,0.2730186780293783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,6144,0.23590826988220215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,5120,0.20131093660990396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,2560,0.11041279633839925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,4096,0.16556159655253094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,3584,0.14732160568237304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,3072,0.12856000264485676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,2048,0.09179946581522623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,1536,0.06948586304982504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,768,0.03607893387476603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,1024,0.04626880089441936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,512,0.028309333324432372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,256,0.023280000686645506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,128,0.01991893251736959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,64,0.019064533710479736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,32,0.018689066171646118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,8192,0.2953119913736979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,10240,0.3633898735046387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,12288,0.4302666664123535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,65536,2.544952646891276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,16384,0.5702389399210612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,7168,0.2593941370646159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,6144,0.22539839744567872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,5120,0.18977279663085939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,4096,0.15562987327575684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,2048,0.08462293148040771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,3584,0.13934292793273925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,3072,0.12138880093892415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,2560,0.10298879941304524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,1024,0.040141868591308597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,256,0.021409066518147786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,1536,0.06251840194066366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,768,0.03189226587613424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,512,0.027160533269246417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,128,0.018427733580271402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,64,0.017947733402252197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,32,0.01737173398335775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,8192,0.29038718541463215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,10240,0.3590773264567057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,12288,0.4252458572387695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,65536,2.3822677612304686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,16384,0.5650026957194011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,7168,0.2560191949208578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,6144,0.22213013966878256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,5120,0.18628907203674316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,4096,0.1525760014851888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,3584,0.13585707346598308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,3072,0.11793920199076335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,2560,0.10023252964019776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,2048,0.0810314655303955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,256,0.020381865898768108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,1024,0.03852266470591227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,512,0.025634133815765382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,1536,0.05811946789423624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,768,0.030358399947484332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,128,0.017498666048049928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,32,0.01685333251953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,64,0.01690559983253479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,8192,0.28925228118896484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,10240,0.35726079940795896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,12288,0.4245738665262858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,65536,2.2218772888183596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,16384,0.5625269571940105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,7168,0.2548917293548584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,6144,0.2215424060821533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,5120,0.18538880348205566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,4096,0.15109866460164387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,3584,0.13460693359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,2048,0.08037973244984944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,3072,0.11753280162811279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,2560,0.10015679995218914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,768,0.03062506715456645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,256,0.020106667280197145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,1024,0.03687893152236939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,1536,0.05787093242009481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,512,0.02577386697133382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,128,0.017509333292643228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,64,0.016406400005022685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,32,0.016293332974116007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,8192,0.2879701296488444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,10240,0.3570357322692871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,12288,0.42281173070271805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,65536,2.2283681233723955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,16384,0.5624383926391602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,7168,0.25377492904663085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,6144,0.21994986534118652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,5120,0.18462293942769367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,4096,0.1516640027364095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,2048,0.07934292952219645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,3584,0.13368959426879884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,3072,0.1164896011352539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,2560,0.09857707023620606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,1024,0.036108799775441486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,256,0.02011093298594157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,1536,0.05654826561609903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,512,0.025169066588083905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,768,0.03037866751352946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,128,0.01725333333015442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,64,0.016306133071581522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,32,0.01620586713155111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,7168,2.765772755940755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,65536,2.2204010009765627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,8192,3.0943018595377607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,10240,3.8739105224609376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,12288,4.660102335611979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,4096,1.5195935567220054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,5120,1.8879316965738933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,3584,1.4041269938151042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,6144,2.2287946065266926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,3072,1.1823871612548829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,16384,6.280550638834635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,2048,0.8095488230387369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,1024,0.46905921300252273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,1536,0.6510485331217448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,2560,0.9764586766560873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,64,0.21804800033569335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,128,0.2238368034362793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,256,0.24076587359110513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,32,0.21345814069112143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,768,0.39423465728759766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,512,0.30860052108764646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,7168,0.7520064036051433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,65536,2.2056907653808593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,8192,0.8603935877482096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,10240,1.0689216613769532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,12288,1.2803284962972006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,4096,0.44773120880126954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,5120,0.5505002975463867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,3584,0.3940341313680013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,6144,0.6465450922648113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,3072,0.34182399113972983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,16384,1.6960384368896484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,768,0.11959786415100097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,1024,0.1427018642425537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,1536,0.18844374020894367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,512,0.09349760214487711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,2560,0.2913461367289225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,2048,0.23689173062642416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,256,0.0781002680460612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,128,0.07149759928385416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,64,0.07145493030548096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,32,0.0713098684946696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,7168,0.5816490809122722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,8192,0.6594773610432942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,10240,0.8228810628255209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,12288,0.9833578745524088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,5120,0.4203466733296712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,6144,0.5010581334431966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,3584,0.3043701489766439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,16384,1.3154132843017579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,4096,0.34304854075113933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,1024,0.115611735979716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,2048,0.18771519660949706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,1536,0.1531808058420817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,2560,0.22485334078470864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,3072,0.26306559244791666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,512,0.07588053544362386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,768,0.09756693045298258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,32,0.05836693445841471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,128,0.058413867155710855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,256,0.06436906655629476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,64,0.05876906712849935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,8192,0.5752085367838542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,10240,0.7186272303263347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,12288,0.850981330871582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,16384,1.1349098205566406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,7168,0.5069226582845052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,6144,0.4437919934590657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,5120,0.37504746119181315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,4096,0.3044714609781901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,3584,0.2687061309814453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,2560,0.20135787328084312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,3072,0.23622825940450035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,2048,0.16616214116414388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,1536,0.13510826428731282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,1024,0.10366186300913494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,768,0.0877791961034139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,512,0.07109973430633545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,32,0.05161066850026449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,128,0.05180373191833496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,256,0.057411201794942225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,65536,7.172005208333333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,64,0.05192960103352865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,8192,0.4860032081604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,10240,0.6114378611246745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,12288,0.7185781478881836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,65536,5.481488037109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,16384,0.9479775746663412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,6144,0.37582505544026695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,7168,0.4274911880493164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,5120,0.31269334157307943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,4096,0.26002880732218425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,3584,0.22879145940144857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,3072,0.19947627385457356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,2560,0.16937813758850098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,2048,0.14110933939615886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,768,0.07454720338185629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,1024,0.088373335202535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,1536,0.11582720279693604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,256,0.049813334147135416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,512,0.05903253157933554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,128,0.04421226580937703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,64,0.04543999830881755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,32,0.045762133598327634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,8192,0.44484478632609054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,65536,4.581577555338542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,10240,0.5467530568440755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,12288,0.660916264851888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,16384,0.8610890706380209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,4096,0.23642239570617676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,5120,0.2873941421508789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,6144,0.3392106691996256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,7168,0.39230400721232095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,3072,0.18254613876342773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,3584,0.20789440472920737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,2048,0.13060906728108723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,1536,0.10581013361612956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,2560,0.15616426467895508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,1024,0.08087999820709228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,768,0.06839146614074706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,64,0.041611735026041666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,512,0.055544535319010414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,128,0.042606933911641436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,256,0.04630613327026367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,32,0.04256426493326823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,8192,0.39710187911987305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,10240,0.49403521219889324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,12288,0.58516050974528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,65536,3.844735972086588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,16384,0.7724032084147135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,4096,0.21592532793680824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,7168,0.3507765452067057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,6144,0.3077930768330892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,5120,0.25931946436564124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,3584,0.18884159723917643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,3072,0.16560427347819012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,1536,0.09606293042500814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,2048,0.11792106628417968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,2560,0.13939733505249025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,768,0.06349013249079385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,1024,0.07388160228729249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,512,0.051420799891153966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,64,0.03974719842274983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,256,0.04189653396606445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,128,0.03841919898986816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,32,0.038970665136973066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,8192,0.35788052876790366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,10240,0.4416042645772298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,12288,0.5260586738586426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,65536,3.4233856201171875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,16384,0.6982048034667969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,4096,0.19349759419759113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,6144,0.27593599955240883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,7168,0.3162410736083984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,5120,0.23530133565266928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,3584,0.17153706550598144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,1536,0.08832639853159586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,2048,0.10691413084665935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,1024,0.06842026710510254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,2560,0.1260981321334839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,3072,0.14958720207214354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,256,0.03787306547164917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,512,0.047917866706848146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,64,0.034355199337005614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,128,0.03442666530609131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,768,0.05847040017445883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,32,0.034151466687520345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,8192,0.3149770736694336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,10240,0.3863744099934896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,65536,3.123223368326823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,12288,0.45595305760701493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,16384,0.6056277592976888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,7168,0.2777642567952474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,6144,0.24243733088175454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,4096,0.16784532864888507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,5120,0.20890026092529296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,3584,0.1510111967722575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,3072,0.13070399761199952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,1536,0.07883093357086182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,2048,0.09414827028910319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,2560,0.11170026461283367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,512,0.043169065316518145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,768,0.05158186753590902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,1024,0.060779734452565515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,128,0.030230400959650676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,256,0.034953598181406656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,64,0.030109866460164385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,32,0.030057599147160847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,8192,0.30417919158935547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,10240,0.37632745107014975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,65536,2.736219787597656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,12288,0.44020268122355144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,16384,0.5818485260009766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,7168,0.2685930569966634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,4096,0.16213653882344564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,5120,0.20043306350708007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,6144,0.23504640261332194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,3584,0.14528106053670248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,3072,0.12586666742960612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,1536,0.0766314665476481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,2048,0.0936896006266276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,2560,0.1083072026570638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,1024,0.05818773508071899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,512,0.04091946681340535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,256,0.032868266105651855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,768,0.04987733364105225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,128,0.027908267577489217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,32,0.029333333174387615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,64,0.029368533690770464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,8192,0.27151254018147786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,10240,0.3360469182332357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,65536,2.3605504353841145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,12288,0.3930762608846029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,16384,0.5208767890930176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,7168,0.2403872013092041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,5120,0.17745812733968097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,6144,0.21157439549763998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,4096,0.14598080317179363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,3584,0.1297706683476766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,1536,0.06819626490275064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,2048,0.08420480092366536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,2560,0.09830933411916097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,3072,0.11332480112711589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,512,0.037427198886871335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,256,0.029864533742268877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,768,0.04504426717758179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,1024,0.05264960130055746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,128,0.027386667331059773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,64,0.02665173411369324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,32,0.02648213307062785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,8192,0.24707199732462565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,10240,0.3070495923360189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,65536,2.2825663248697916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,12288,0.36169811884562175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,16384,0.4759541193644206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,4096,0.13324906826019287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,5120,0.16259733835856122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,6144,0.1924821376800537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,7168,0.21872852643330892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,1536,0.06113919814427694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,2048,0.07595733006795248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,3584,0.11851627031962078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,2560,0.08945066928863525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,3072,0.1042944033940633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,768,0.03879253466924031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,1024,0.046070400873819986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,512,0.03289706707000732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,256,0.025587199131647746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,128,0.022486400604248048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,64,0.023171200354894003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,32,0.022424532969792684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,8192,0.22440640131632486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,10240,0.27971200942993163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,65536,2.020746612548828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,12288,0.32968959808349607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,16384,0.43353706995646163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,6144,0.1737738609313965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,4096,0.1211466630299886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,7168,0.1985589345296224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,5120,0.1472490628560384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,3584,0.108570663134257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,1536,0.054749866326649986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,3072,0.0960693359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,2048,0.06817813714345297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,2560,0.08169706662495932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,512,0.029604266087214153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,128,0.02156160076459249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,256,0.02297919988632202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,1024,0.040812798341115314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,768,0.034481068452199296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,64,0.02159893314043681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,32,0.0214848001797994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,8192,0.2042293389638265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,10240,0.25343039830525715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,65536,1.8464789072672527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,12288,0.2990784009297689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,16384,0.3936554590861002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,4096,0.11235413551330567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,6144,0.15980159441630046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,5120,0.1355786641438802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,7168,0.18103787104288738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,3584,0.10035093625386556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,2560,0.07453760306040445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,1536,0.04746346473693848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,2048,0.060725335280100504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,3072,0.08756053447723389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,1024,0.03602026700973511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,768,0.030805333455403643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,32,0.020062933365503945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,256,0.021785600980122884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,512,0.026550400257110595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,128,0.020129066705703736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,64,0.019688532749811808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,8192,0.18432532946268718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,65536,1.6928853352864581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,10240,0.22466346422831215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,12288,0.26603946685791013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,16384,0.34783894220987954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,6144,0.1418783982594808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,7168,0.16349120140075685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,5120,0.1222314675649007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,4096,0.10033280054728191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,3072,0.07746559778849284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,1536,0.04088213443756104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,2048,0.05280640125274658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,2560,0.06479359865188598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,3584,0.08957760334014893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,1024,0.03208213249842326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,256,0.020460800329844157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,512,0.02404159903526306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,128,0.018787199258804323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,64,0.018611200650533042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,768,0.027169066667556762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,32,0.01834986607233683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,65536,1.5248308817545573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,8192,0.1745397408803304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,10240,0.21209920247395836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,12288,0.25038506189982096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,16384,0.3272085189819336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,4096,0.09531839688618979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,5120,0.11598400274912517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,6144,0.13582080205281574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,7168,0.15517973899841309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,3072,0.07158506711324056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,3584,0.08339946269989014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,1536,0.03771946827570598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,2048,0.04726080099741618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,2560,0.05952426592508951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,1024,0.029759999116261798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,256,0.019684267044067384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,512,0.02296746571858724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,128,0.018045866489410402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,768,0.026260266701380413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,64,0.017755732933680216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,32,0.017612799008687337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,65536,1.3523551940917968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,8192,0.16325759887695312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,10240,0.19936319986979167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,12288,0.2335584004720052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,16384,0.30854612986246743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,5120,0.10854079723358154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,6144,0.12746559778849284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,7168,0.14481706619262696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,4096,0.08822080294291178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,3584,0.07785279750823974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,2560,0.05441173315048218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,1536,0.034332799911499026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,2048,0.042685866355895996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,3072,0.06668053468068441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,256,0.01882773240407308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,128,0.01723626653353373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,512,0.021767467260360718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,1024,0.028500266869862872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,768,0.024884267648061117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,64,0.017203199863433837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,32,0.016845866044362386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,65536,1.2708170572916666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,8192,0.15711894035339355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,10240,0.19141866366068522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,12288,0.227728001276652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,16384,0.2977823893229167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,7168,0.13953280448913574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,6144,0.12178986867268879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,4096,0.0852831999460856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,5120,0.10300266742706299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,3584,0.07401386896769205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,1536,0.032013867298762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,2048,0.039280001322428384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,2560,0.05019520123799642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,3072,0.06236053307851156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,512,0.020887466271718343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,256,0.017816533644994102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,768,0.0236629327138265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,128,0.0165994664033254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,1024,0.026343466838200887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,32,0.016229333480199178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,64,0.016318933169047038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,65536,1.1924554189046224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,8192,0.15355733235677083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,10240,0.18804052670796711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,12288,0.2225045363108317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,16384,0.2907338778177897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,7168,0.1368597348531087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,5120,0.10180586973826092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,6144,0.11885546843210856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,4096,0.08228480021158854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,2560,0.0478218674659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,3584,0.07148693402608236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,2048,0.03818879922231038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,1536,0.03137279947598775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,3072,0.05952426592508951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,1024,0.025830399990081788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,768,0.02304960091908773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,128,0.016102400422096253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,256,0.017518933614095053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,512,0.02045546571413676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,64,0.01608746647834778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,32,0.015997866789499916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,65536,1.1255850474039713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,8192,0.15238399505615235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,10240,0.1866538683573405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,12288,0.2205066680908203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,16384,0.29341227213541665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,7168,0.13378346761067708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,5120,0.09969706535339355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,4096,0.0799285332361857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,6144,0.11761279900868733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,2560,0.04595839977264404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,1536,0.030744532744089764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,3584,0.06972053050994872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,3072,0.05758399963378906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,2048,0.03750506639480591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,1024,0.02534186641375224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,128,0.016065067052841185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,64,0.01588586668173472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,768,0.02284053365389506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,256,0.017130666971206666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,512,0.020473599433898926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,32,0.015498666961987814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,65536,1.1167466481526693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,8192,0.15175360043843586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,10240,0.1863093376159668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,12288,0.21957012812296547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,16384,0.29082132975260416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,7168,0.13368213971455892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,6144,0.11664213339487713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,5120,0.09927573204040527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,4096,0.07976319789886474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,3072,0.056651735305786134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,3584,0.06887253125508627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,2560,0.04537493387858073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,1024,0.025217066208521526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,1536,0.030418133735656737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,2048,0.036474665006001786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,768,0.022817067305246987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,256,0.017364267508188883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,512,0.020091732343037925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,128,0.016090666254361473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,64,0.015869866808255514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,32,0.015752533078193666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,65536,1.1168810526529946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,7168,1.3024608612060546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,8192,1.4930197397867837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,6144,1.1229525248209635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,10240,1.8640416463216145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,65536,1.1144395192464194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,12288,2.253973388671875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,4096,0.7699349085489909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,3584,0.6762186686197917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,5120,0.9411296208699544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,3072,0.5883904139200846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,2560,0.49863039652506513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,2048,0.40981760025024416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,256,0.13224746386210123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,1536,0.32683839797973635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,768,0.20537707010904946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,16384,3.008586629231771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,1024,0.24287679990132652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,64,0.1210250695546468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,128,0.1221237341562907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,512,0.16558079719543456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,32,0.11871893405914306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,6144,0.32281812032063806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,7168,0.37020371754964193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,8192,0.4206837336222331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,10240,0.5213856061299642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,12288,0.616099230448405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,4096,0.22232747077941895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,3584,0.19885120391845704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,3072,0.17375359535217286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,5120,0.27159039179484046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,16384,0.8365503946940104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,2560,0.15157546997070312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,1024,0.07985280354817709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,1536,0.10420479774475097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,768,0.06888319651285807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,128,0.04434026479721069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,2048,0.12732053597768148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,512,0.05563413302103678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,256,0.04847573439280192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,64,0.04543253183364868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,32,0.04558826684951782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,7168,0.2957930564880371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,8192,0.3333706537882487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,10240,0.41074453989664717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,12288,0.4984789212544759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,16384,0.6474090576171875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,3584,0.16057813962300618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,4096,0.17874773343404132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,5120,0.2162048021952311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,6144,0.2561855951944987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,2560,0.12267519632975261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,1536,0.0847978671391805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,1024,0.06658133268356323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,2048,0.1036906639734904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,768,0.057036801179250085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,3072,0.1411242643992106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,512,0.04751040140787761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,256,0.04107306798299153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,128,0.03799253304799398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,64,0.03831040064493815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,32,0.03952639897664388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,8192,0.294702943166097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,10240,0.36755520502726235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,12288,0.4313674608866374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,16384,0.5713151931762696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,6144,0.2267840067545573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,5120,0.19024853706359862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,7168,0.2597386678059896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,4096,0.15805013974507648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,3584,0.14337706565856934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,3072,0.12596906820933024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,2560,0.10892480214436848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,65536,3.259708913167318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,1536,0.07674667040506998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,2048,0.09282240072886148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,1024,0.06010239919026693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,768,0.05240319967269898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,256,0.03628480037053426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,512,0.046641067663828535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,64,0.03350186745325724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,128,0.033885868390401204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,32,0.034016001224517825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,65536,2.5788853963216143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,8192,0.24744532903035482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,10240,0.30572373072306314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,12288,0.3609439849853516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,16384,0.4774880091349284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,7168,0.22216320037841797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,6144,0.18919359842936198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,5120,0.16174933115641277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,4096,0.1340640068054199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,3072,0.10872639815012615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,3584,0.12311893304189045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,1024,0.052892800172170004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,2560,0.09268159866333008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,1536,0.06574079990386963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,2048,0.0798367977142334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,768,0.046401067574818926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,512,0.039451734224955244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,128,0.02943039933840434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,256,0.0324725329875946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,64,0.029909332593282063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,32,0.030510934193929036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,65536,2.2627817789713545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,8192,0.22611625989278158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,10240,0.275655460357666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,7168,0.19916693369547528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,12288,0.327890141805013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,16384,0.43126506805419923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,6144,0.1721717357635498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,5120,0.1437664031982422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,4096,0.12031253178914388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,3584,0.10609920024871826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,3072,0.09348800182342529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,2560,0.08326613108317057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,1536,0.0572703997294108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,1024,0.04437013467152913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,2048,0.06951146920522054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,768,0.0385098656018575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,256,0.02927253246307373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,512,0.03386346499125163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,128,0.027422932783762614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,64,0.026909865935643512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,32,0.02832000056902568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,65536,1.9049524943033855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,8192,0.20437119801839193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,10240,0.25215892791748046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,7168,0.18470293680826824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,12288,0.2984821319580078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,16384,0.39329067866007483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,4096,0.11390293439229329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,6144,0.15708907445271808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,5120,0.13444587389628093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,3584,0.10074559847513835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,3072,0.0914901336034139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,2560,0.08036373456319174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,2048,0.0679306666056315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,1536,0.056927998860677086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,1024,0.04487786690394084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,512,0.03426880041758219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,768,0.04010239839553833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,256,0.02860586643218994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,128,0.027258666356404622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,64,0.027081600824991864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,32,0.026423466205596925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,65536,1.7157044728597004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,8192,0.1858261267344157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,10240,0.22626026471455893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,7168,0.1642730712890625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,12288,0.26782614390055337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,16384,0.34906028111775717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,6144,0.14086826642354328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,5120,0.11706986427307128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,4096,0.09974719683329264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,3584,0.08920000394185384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,3072,0.07955199877421061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,1536,0.05039039850234985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,2560,0.0712394634882609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,1024,0.03958186705907186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,2048,0.06012479861577352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,512,0.029575467109680176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,768,0.03399680058161418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,256,0.025552000602086383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,128,0.022705066204071044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,64,0.022693334023157756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,32,0.022794665892918904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,65536,1.5248405456542968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,8192,0.16245120366414387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,10240,0.1969055970509847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,12288,0.23311999638875328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,7168,0.14077547391255696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,16384,0.30888427098592125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,6144,0.1239573319753011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,5120,0.10452480316162109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,3584,0.08008960088094076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,4096,0.08826026916503907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,3072,0.07141013145446777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,2560,0.06256959835688272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,1536,0.042418134212493894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,1024,0.03419839938481649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,512,0.026048000653584796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,2048,0.0518666664759318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,768,0.03025919993718465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,64,0.02132373253504435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,256,0.022318933407465616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,128,0.02140480081240336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,32,0.02118933399518331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,65536,1.364346694946289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,8192,0.15695892969767253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,10240,0.19464426040649413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,7168,0.14046400388081867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,12288,0.23057066599527992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,16384,0.3029343922932943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,6144,0.12070826689402263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,3584,0.07844266891479493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,4096,0.08783253033955893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,5120,0.10459946791330973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,3072,0.06990613142649332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,768,0.030139732360839843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,1536,0.04254613320032756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,1024,0.03376106818517049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,2048,0.050735998153686526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,2560,0.06003413200378418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,512,0.02518293261528015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,64,0.02104640007019043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,256,0.022685867547988892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,128,0.02093440095583598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,32,0.02096959948539734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,65536,1.1728543599446615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,7168,0.12259413401285808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,8192,0.13974612553914387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,10240,0.17225279808044433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,12288,0.20493866602579752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,16384,0.2688533465067545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,4096,0.07821333408355713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,5120,0.09253333409627279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,6144,0.10692266623179118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,3584,0.07084373633066812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,3072,0.06227306524912516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,1536,0.03720959822336833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,2048,0.04501760005950928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,2560,0.053597867488861084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,1024,0.0308351993560791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,768,0.027407999833424884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,512,0.023089067141215006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,64,0.019821866353352865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,128,0.019720532496770225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,256,0.020820266008377074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,65536,1.1626324971516928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,32,0.0197653333346049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,8192,0.13009706338246663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,7168,0.11597013473510742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,10240,0.15899945894877116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,12288,0.18800959587097169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,6144,0.10068906943003338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,16384,0.24609386126200355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,5120,0.0856544017791748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,4096,0.07176853020985921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,3584,0.06449813445409139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,3072,0.0562613328297933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,2048,0.04135893185933431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,2560,0.04847040176391602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,1024,0.02892586588859558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,1536,0.03509013255437215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,768,0.02566933234532674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,512,0.02333866755167643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,256,0.020444800456364952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,32,0.01926079988479614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,65536,1.0369344075520834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,128,0.019206400712331137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,64,0.01927786668141683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,7168,0.10550186634063721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,8192,0.11931413014729816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,10240,0.14541652997334797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,12288,0.17073067029317218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,16384,0.22378239631652833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,5120,0.07849600315093994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,6144,0.09188266595204672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,3584,0.05732160011927286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,4096,0.0649397333463033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,1536,0.03181653420130412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,3072,0.05029653310775757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,2560,0.043185067176818845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,2048,0.03744639952977498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,1024,0.026257065931955974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,64,0.01830293337504069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,768,0.023797333240509033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,512,0.02169493238131205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,256,0.019476266702016194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,128,0.018205867211023966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,32,0.018296533823013307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,65536,0.9228309631347656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,8192,0.10953813393910725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,7168,0.09690346717834472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,10240,0.1328266700108846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,12288,0.15574293136596679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,16384,0.20374933878580728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,6144,0.08464319705963134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,5120,0.0713045358657837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,4096,0.058023468653361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,3584,0.05138453245162964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,3072,0.04530880053838094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,2560,0.03943253358205159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,2048,0.03396906852722168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,512,0.020683733622233073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,1024,0.02513386607170105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,1536,0.02826346755027771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,768,0.02262079914410909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,128,0.017577600479125977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,256,0.018253866831461588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,64,0.01758400003115336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,32,0.017577600479125977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,65536,0.8413386662801108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,7168,0.08797226746877035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,8192,0.09857280254364013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,10240,0.11975786685943604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,12288,0.1396725336710612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,16384,0.1815008004506429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,6144,0.07553386688232422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,3072,0.03938026825586955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,3584,0.04444479942321777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,4096,0.05023253361384074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,5120,0.06304426590601603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,2048,0.030322132507960002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,1536,0.026678399244944258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,1024,0.023004800081253052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,2560,0.035070931911468504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,768,0.021171200275421142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,512,0.019290665785471596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,256,0.017722666263580322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,64,0.01681173245112101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,128,0.016910932461420693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,65536,0.7686304092407227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,32,0.016884267330169678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,7168,0.08496106465657552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,8192,0.09561493396759033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,10240,0.11880959669748943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,12288,0.13680853843688964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,16384,0.1727178732554118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,5120,0.060811734199523924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,6144,0.07346239884694418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,3584,0.041816532611846924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,4096,0.04751253525416056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,2560,0.03250773350397746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,3072,0.03735893170038859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,768,0.020456532637278237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,2048,0.030158933003743487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,65536,0.6837120056152344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,1536,0.02553066611289978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,1024,0.022603732347488404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,512,0.018733867009480796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,256,0.017203199863433837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,128,0.016434133052825928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,32,0.01641706625620524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,64,0.01644266645113627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,7168,0.08196266492207846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,12288,0.13284906546274822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,8192,0.09258453051249185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,10240,0.1140874703725179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,16384,0.1688202699025472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,5120,0.05682133436203003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,4096,0.04523413181304932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,6144,0.07051520347595215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,3584,0.04005866845448812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,3072,0.03572053511937459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,2560,0.03215679923693339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,1536,0.02486506700515747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,2048,0.028032000859578448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,768,0.019745065768559774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,1024,0.0214954674243927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,65536,0.6574122746785481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,512,0.01851946711540222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,256,0.016806399822235106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,128,0.016221867005030314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,64,0.015879467129707336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,32,0.016059733430544534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,8192,0.0862879991531372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,7168,0.07562560240427653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,10240,0.10735680262247722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,16384,0.15962986946105956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,12288,0.12483839988708496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,3584,0.0365503986676534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,3072,0.0325984001159668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,4096,0.0415338675181071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,6144,0.06348906755447388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,5120,0.0510047992070516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,2560,0.029587199290593464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,1024,0.020577067136764528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,1536,0.023449599742889404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,2048,0.026637866099675493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,768,0.019092265764872232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,65536,0.6137119928995769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,512,0.018038400014241538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,256,0.016457600394884746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,128,0.01581439971923828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,64,0.01586133340994517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,32,0.015795200069745382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,7168,0.0740447998046875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,8192,0.08419626553853353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,10240,0.1031488021214803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,12288,0.12103253205617268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,16384,0.15749227205912272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,5120,0.048190931479136154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,3072,0.031923200686772665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,4096,0.03818133274714152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,6144,0.06158080101013184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,3584,0.03453973531723022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,65536,0.5806293487548828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,2048,0.02606613238652547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,768,0.018744534254074095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,1536,0.02358293334643046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,2560,0.03028800090154012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,1024,0.020629332462946574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,256,0.01641386648019155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,512,0.017787732680638633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,128,0.015664000312487283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,64,0.015497600038846334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,32,0.015656532843907674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,7168,0.0736512025197347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,8192,0.08342506885528564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,10240,0.10314773718516032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,12288,0.11961920261383056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,16384,0.1545109272003174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,4096,0.037330134709676104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,6144,0.06005546649297079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,5120,0.047541332244873044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,3584,0.03433813254038493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,3072,0.03132266600926717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,1024,0.020606933037439983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,65536,0.5733343760172527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,1536,0.023215999205907188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,2048,0.026255999008814496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,2560,0.028568534056345622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,512,0.01731946667035421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,768,0.018962132930755615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,128,0.015664000312487283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,256,0.016153599818547568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,32,0.01516800026098887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,64,0.015545599659283958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,7168,0.07215039730072022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,8192,0.08306026458740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,10240,0.1017077366511027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,12288,0.1193930705388387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,16384,0.15350079536437988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,6144,0.06006186803181966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,5120,0.04761706590652466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,3072,0.03126186728477478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,3584,0.03383040030797323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,4096,0.037137067317962645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,1536,0.02322346568107605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,65536,0.5657290776570638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,1024,0.020492800076802573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,2560,0.028725334008534747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,2048,0.02570880055427551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,768,0.01888213356335958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,512,0.017628800868988038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,128,0.01588159998257955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,256,0.01599253316720327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,32,0.015290666619936624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,64,0.015565866231918335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,65536,0.5661951700846355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,6144,0.5849589029947917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,7168,0.6720885594685873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,8192,0.7680053075154623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,5120,0.4992533365885417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,10240,0.9541727701822916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,12288,1.1689834594726562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,4096,0.40476051966349286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,3584,0.3597471872965495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,3072,0.31201279958089195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,2048,0.22151039441426595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,2560,0.26553279558817544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,1024,0.13479894002278645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,768,0.11472853024800618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,1536,0.18134400049845378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,512,0.08744426568349203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,256,0.07731520334879557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,128,0.07190933227539062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,64,0.07289493083953857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,32,0.07145280043284098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,16384,1.5259595235188803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,7168,0.18817386627197266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,10240,0.26308693885803225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,8192,0.2134058634440104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,6144,0.17230933507283527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,12288,0.31133759816487633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,4096,0.12127466996510823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,16384,0.4126592000325521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,3072,0.09825920263926188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,3584,0.10824746290842693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,5120,0.1430005391438802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,2560,0.0841759999593099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,2048,0.07238399982452393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,512,0.03748586575190226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,1536,0.06027306715647379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,768,0.043570133050282796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,1024,0.0494976003964742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,256,0.03232853412628174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,128,0.02946346600850423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,64,0.02985386649767558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,32,0.031022934118906657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,10240,0.20952213605244957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,7168,0.15293013254801432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,8192,0.16935466130574545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,12288,0.24923413594563804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,16384,0.3311925252278646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,6144,0.13465280532836915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,4096,0.09749226570129395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,3584,0.08910293579101562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,5120,0.1166922648747762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,2560,0.06958186626434326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,3072,0.08025173346201578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,1536,0.05140693187713623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,2048,0.05965439875920614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,1024,0.041943466663360594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,768,0.037189332644144694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,32,0.025969066222508747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,256,0.028657066822052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,128,0.026383999983469648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,512,0.03299733400344849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,64,0.026549333333969118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,8192,0.14939200083414714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,10240,0.18504853248596193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,12288,0.21580266952514648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,16384,0.2885077476501465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,65536,1.6506655375162762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,5120,0.10081173578898113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,6144,0.11937706470489502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,7168,0.1333461284637451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,4096,0.0853226661682129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,3584,0.08222400347391765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,3072,0.06907413005828858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,2560,0.06207786798477173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,2048,0.0532533327738444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,1536,0.04517546494801839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,1024,0.036210131645202634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,768,0.03203840057055156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,64,0.022392533222834268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,512,0.028508800268173217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,256,0.024877866109212242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,128,0.023091200987497965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,32,0.02248959938685099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,65536,1.2688223520914712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,10240,0.15307626724243165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,8192,0.1242410659790039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,12288,0.18243199984232586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,7168,0.11301759878794353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,16384,0.24326933224995934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,3584,0.06668053468068441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,5120,0.0861461321512858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,6144,0.09923413594563803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,4096,0.073744002978007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,3072,0.060134398937225345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,2560,0.05257706642150879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,1536,0.03847039937973022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,2048,0.04494080146153768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,1024,0.031597866614659624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,768,0.02852906584739685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,65536,1.140054448445638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,512,0.025603200991948443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,256,0.02159573237101237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,128,0.020977065960566203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,32,0.02109439969062805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,64,0.021255467335383096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,7168,0.11035733222961426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,8192,0.12540053526560466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,10240,0.14983146985371906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,12288,0.17925012906392415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,16384,0.23426666259765624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,4096,0.07201813062032064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,3584,0.06529920101165772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,5120,0.08635626633961996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,6144,0.09828693072001139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,3072,0.05838826497395834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,2560,0.051173333326975504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,2048,0.044343467553456625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,1024,0.031128533681233722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,65536,0.9452981313069662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,1536,0.03790613412857056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,768,0.028197334210077925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,512,0.02490880091985067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,256,0.022131200631459555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,64,0.02037866711616516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,128,0.02081386645634969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,32,0.021160533030827842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,7168,0.09669439792633057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,10240,0.1306506633758545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,8192,0.10661013126373291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,12288,0.1538335959116618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,16384,0.20248212814331054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,5120,0.07519573370615641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,4096,0.06313493251800537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,3584,0.05737599929173788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,6144,0.08598399957021077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,3072,0.051191465059916175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,2560,0.0453002651532491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,1536,0.033156265815099076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,1024,0.028135466575622558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,2048,0.03898773193359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,512,0.022936532894770302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,65536,0.9107882817586264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,768,0.025564799706141155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,32,0.019683200120925903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,256,0.020869332551956176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,128,0.019448532660802206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,64,0.019466666380564372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,8192,0.09597760041554769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,7168,0.08588053385416666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,12288,0.13773333231608073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,10240,0.11493972937266032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,16384,0.17967573801676434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,4096,0.05717973311742147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,3584,0.05215893189112345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,3072,0.04666879971822103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,6144,0.07716800371805826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,5120,0.06717759768168131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,2560,0.04088426828384399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,2048,0.035702399412790936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,512,0.022180267175038657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,1536,0.030741333961486816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,1024,0.026834134260813398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,768,0.024399999777475992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,65536,0.7798890431722005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,128,0.019091200828552247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,256,0.020096000035603842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,64,0.019066667556762694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,32,0.01931519905726115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,7168,0.0772437334060669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,8192,0.08377172946929931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,12288,0.11972053050994873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,10240,0.10021333694458008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,16384,0.15750400225321454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,4096,0.049762133757273355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,3584,0.04429440100987752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,6144,0.06900693575541178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,5120,0.059129599730173746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,3072,0.0400607983271281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,2560,0.03556160132090251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,65536,0.6860074361165365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,1024,0.023465599616368613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,2048,0.032153600454330446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,1536,0.028202666838963823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,768,0.021915733814239502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,512,0.02035413384437561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,256,0.018961066007614137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,64,0.018252799908320107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,128,0.017828265825907387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,32,0.018234666188557944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,8192,0.08199040095011392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,7168,0.0757215976715088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,12288,0.11701653003692628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,10240,0.09898239771525065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,16384,0.15470080375671386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,6144,0.06641173362731934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,5120,0.05867520173390707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,4096,0.046036267280578615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,3584,0.04170133272806804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,3072,0.03764479955037435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,65536,0.5901621500651042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,1024,0.022395733992258707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,1536,0.026757333676020307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,768,0.021031467119852702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,2560,0.03402133385340373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,2048,0.030010666449864703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,512,0.019317332903544107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,64,0.018168532848358156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,256,0.018288000424702962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,128,0.017794134219487508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,32,0.01804479956626892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,8192,0.0755733331044515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,7168,0.06759040355682373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,10240,0.08946986993153891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,12288,0.10496319929758709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,16384,0.13820907274882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,5120,0.05206933418909708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,6144,0.06068373521169027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,4096,0.04354133208592732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,3584,0.03930240074793498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,65536,0.5733631769816081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,2048,0.029267199834187824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,1536,0.025913600126902265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,2560,0.032278400659561154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,3072,0.03630826473236084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,1024,0.02198186715443929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,512,0.019401599963506065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,768,0.02065066695213318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,256,0.018263467152913413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,128,0.017131733894348144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,32,0.017463467518488564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,64,0.017511467138926186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,8192,0.07149759928385416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,7168,0.06442559957504272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,10240,0.08373226324717203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,12288,0.09844373067220052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,6144,0.05744853417078653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,16384,0.12924799919128419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,4096,0.041137067476908366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,3584,0.03796159823735555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,5120,0.049675734837849934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,65536,0.5206389427185059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,3072,0.0348416010538737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,2560,0.03164693315823873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,512,0.019627734025319418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,2048,0.02839679916699727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,1536,0.025224532683690386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,1024,0.021357866128285725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,768,0.02105813423792521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,128,0.017292799552281697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,256,0.018212266763051353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,64,0.01735466718673706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,32,0.017299199104309083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,12288,0.08940906524658203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,7168,0.058730665842692056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,10240,0.07692800362904867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,8192,0.0645034670829773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,16384,0.11717653274536133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,6144,0.050986667474110924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,3584,0.0337557315826416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,4096,0.03697173198064168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,5120,0.04347519874572754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,65536,0.4959541320800781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,3072,0.03141973416010539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,2560,0.028689066569010418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,1024,0.020377600193023683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,2048,0.025573333104451496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,1536,0.023205333948135377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,768,0.01932800014813741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,512,0.018359466393788656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,256,0.017401599884033205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,128,0.016745599110921223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,64,0.016769067446390788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,32,0.01697280009587606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,12288,0.0846880038579305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,7168,0.051787734031677246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,8192,0.05826773246129354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,10240,0.07123200098673502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,16384,0.10683733622233074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,6144,0.04549119869867961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,65536,0.4296138763427734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,5120,0.03919253349304199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,4096,0.03345706860224406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,3584,0.03131733338038127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,3072,0.029208532969156903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,2560,0.026630399624506633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,2048,0.02405866583188375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,256,0.016935465733210246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,768,0.01875200072924296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,1024,0.01951040029525757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,1536,0.0218560000260671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,512,0.01788053313891093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,64,0.016451199849446617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,32,0.016356266538302102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,128,0.016247466206550598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,10240,0.06930987040201822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,12288,0.08284053007761637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,8192,0.053650132815043126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,16384,0.10389866828918456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,7168,0.04837546745936076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,65536,0.404910945892334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,4096,0.03231786688168843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,6144,0.042444801330566405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,3584,0.029677865902582805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,3072,0.02683093349138896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,5120,0.036874667803446455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,2560,0.024998400608698526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,256,0.016461867094039916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,2048,0.022882133722305298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,1024,0.018963199853897095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,1536,0.021061333020528157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,768,0.017875200510025023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,512,0.017369600137074788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,128,0.01576533317565918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,64,0.015833600362141927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,32,0.01590826710065206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,10240,0.06530026594797769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,12288,0.0781599998474121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,7168,0.045006934801737467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,8192,0.050836265087127686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,16384,0.09971733093261718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,65536,0.35674880345662435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,6144,0.040218667189280195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,4096,0.030828799804051715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,3584,0.02836373249689738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,5120,0.035843201478322345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,2560,0.024197334051132204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,2048,0.022130133708318074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,3072,0.025642667214075727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,256,0.01634880006313324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,1024,0.01858560045560201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,1536,0.020284799734751384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,512,0.016771199305852254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,768,0.017684266964594523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,128,0.015621333320935567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,64,0.015738667050997416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,32,0.015783466895421348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,8192,0.04777599970499675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,12288,0.07393279870351156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,7168,0.04197653134663899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,10240,0.06036800146102905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,65536,0.3385439872741699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,16384,0.09529066880544027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,6144,0.03764373461405436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,4096,0.028613332907358808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,3072,0.02518826723098755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,2560,0.022959999243418374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,3584,0.026763733228047686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,5120,0.03336639801661174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,1024,0.018201599518458046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,2048,0.021747199694315593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,768,0.017166932423909508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,1536,0.019814399878184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,256,0.016102400422096253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,512,0.016833066940307617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,128,0.015656532843907674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,32,0.015549866358439126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,64,0.01567466656366984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,7168,0.039459200700124104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,8192,0.04374186595280965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,12288,0.06948800086975097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,10240,0.05623146692911783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,65536,0.3173578580220541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,16384,0.09106986522674561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,3584,0.02632960081100464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,5120,0.03173226714134216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,6144,0.03559786478678385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,4096,0.02791999975840251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,3072,0.024945066372553507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,2560,0.023176532983779908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,2048,0.021592533588409422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,1024,0.018127999703089395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,1536,0.01965013345082601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,512,0.016798933347066246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,768,0.017280000448226928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,256,0.015986133615175882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,128,0.015498666961987814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,64,0.0153546671072642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,32,0.015335466464360556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,7168,0.03876906633377075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,65536,0.303218142191569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,8192,0.04215893348058065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,10240,0.05220799843470255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,12288,0.06344746748606364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,16384,0.08620479901631674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,4096,0.027371732393900554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,3584,0.025906133651733398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,5120,0.030359466870625813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,6144,0.0333024005095164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,3072,0.02434773246447245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,2560,0.02248426675796509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,2048,0.02094399929046631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,1536,0.01951573292414347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,768,0.016960000991821288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,1024,0.017961599429448447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,512,0.016505600015322367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,256,0.01560426652431488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,128,0.01535040040810903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,64,0.015268266201019287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,32,0.01509119967619578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,7168,0.036739198366800944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,8192,0.04053653478622436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,10240,0.05042879978815714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,65536,0.2953354517618815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,12288,0.061936000982920326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,16384,0.0836575984954834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,4096,0.027036799987157183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,5120,0.03014933268229167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,6144,0.03366080125172933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,3584,0.02595626711845398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,3072,0.02409600019454956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,2560,0.02255893349647522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,2048,0.02100480000178019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,1536,0.01950826644897461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,1024,0.01734506686528524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,768,0.01717653274536133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,512,0.01635199983914693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,256,0.015796266992886863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,128,0.015366400281588236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,64,0.015372799833615622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,32,0.015238400300343832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,7168,0.03674986759821574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,8192,0.039655466874440506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,10240,0.048869331677754715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,65536,0.2935306549072266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,12288,0.05981013377507528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,16384,0.08262720108032226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,4096,0.02696746587753296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,5120,0.030189865827560426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,3584,0.02542399962743123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,6144,0.03397546609242757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,2560,0.02248426675796509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,3072,0.024081067244211832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,2048,0.021001599232355752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,1536,0.01918720006942749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,1024,0.017684266964594523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,768,0.016658133268356322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,512,0.016301866372426352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,256,0.015707733233769734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,128,0.01532906691233317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,64,0.01520853340625763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,32,0.015040000279744467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,65536,0.291538143157959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,6144,0.4383104006449382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,5120,0.3689802805582682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,7168,0.5184917449951172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,8192,0.5794453303019206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,10240,0.7168031692504883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,3072,0.23866880734761559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,2560,0.20236694018046059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,4096,0.30237226486206054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,3584,0.2670048077901205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,12288,0.8573269526163736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,2048,0.1700544039408366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,1024,0.10287573337554931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,768,0.08859519958496094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,512,0.07308479944864908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,256,0.07038613160451254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,64,0.059735465049743655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,1536,0.1385994593302409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,128,0.059164798259735106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,32,0.06011413335800171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,16384,1.1182730356852213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,7168,0.14808640480041504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,8192,0.1666591962178548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,10240,0.2032927989959717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,12288,0.2372447967529297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,6144,0.13184213638305664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,16384,0.31341654459635415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,4096,0.09589333534240722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,5120,0.11320213476816814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,3584,0.08669546445210775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,3072,0.07686506907145182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,2560,0.06693546772003174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,512,0.03285013238588969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,1024,0.041230932871500654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,768,0.03711893161137898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,1536,0.05014400084813436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,2048,0.05881066719690958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,256,0.028279467423756914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,128,0.027782400449117024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,64,0.028653866052627562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,32,0.026943999528884887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,7168,0.11819626490275066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,8192,0.13089919884999593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,10240,0.15863359769185384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,12288,0.18521173795064289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,16384,0.24649279912312827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,4096,0.07541226545969645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,5120,0.08892479737599691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,6144,0.10403093496958415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,3584,0.06775893370310465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,2560,0.05446613232294718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,2048,0.04760426680246989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,1536,0.04007039864857991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,1024,0.032995200157165526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,3072,0.06155733267466227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,768,0.029808000723520918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,256,0.023638399442036946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,512,0.02688213388125102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,128,0.021784534056981407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,64,0.021885865926742555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,32,0.021900800863901775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,8192,0.11723732948303223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,10240,0.14201173782348633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,12288,0.16612693468729656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,16384,0.21937813758850097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,7168,0.10460373560587566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,65536,1.2249407450358072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,5120,0.08082239627838135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,6144,0.09279359976450602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,4096,0.06904640197753906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,3584,0.062334934870402016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,3072,0.05866666634877523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,2560,0.04952853520711263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,1024,0.030605866511662798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,1536,0.03698879877726237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,2048,0.043329068024953205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,768,0.02845226724942525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,256,0.026500266790390015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,512,0.02498133381207784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,128,0.02249600092569987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,32,0.023890133698781332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,64,0.02303253412246704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,65536,0.9600298563639322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,7168,0.09391252994537354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,8192,0.10021653175354003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,10240,0.1215178648630778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,12288,0.14062612851460773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,16384,0.18369706471761066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,3584,0.05389973322550455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,4096,0.06037546793619791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,3072,0.04816426833470662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,5120,0.07016212940216064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,6144,0.081878399848938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,2560,0.042770131429036455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,65536,0.8651072184244791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,1536,0.031890134016672775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,2048,0.037885868549346925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,1024,0.027486934264500933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,768,0.02573653260866801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,256,0.02004479964574178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,512,0.02302079995473226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,128,0.019466666380564372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,64,0.019857066869735717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,32,0.019821866353352865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,7168,0.0828437328338623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,8192,0.09273599783579509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,10240,0.10980373223622639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,12288,0.1288149356842041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,16384,0.1738752047220866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,3584,0.0497322678565979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,4096,0.05437333186467489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,3072,0.04514133135477702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,5120,0.06504853169123331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,6144,0.0738432010014852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,65536,0.7235935846964519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,1536,0.029916799068450926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,2560,0.03983786503473918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,2048,0.034587732950846356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,1024,0.025590399901072185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,768,0.023668267329533896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,256,0.020786132415135702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,512,0.022218666474024453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,128,0.01948266625404358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,64,0.01946773330370585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,32,0.019921066363652547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,7168,0.07561919689178467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,8192,0.08206613063812256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,10240,0.09947520097096761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,12288,0.11487893263498943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,16384,0.1535978635152181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,5120,0.05814826488494873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,4096,0.050215466817220056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,3584,0.045525332291920975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,3072,0.041381335258483885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,6144,0.06747840245564779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,65536,0.6556394577026368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,2048,0.03250453273455302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,1536,0.028647466500600176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,1024,0.024894932905832924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,2560,0.036484265327453615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,768,0.02336960037549337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,64,0.018910932540893554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,256,0.01937920053799947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,128,0.018644267320632936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,32,0.018603734175364175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,512,0.021460266908009847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,8192,0.07595199743906657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,7168,0.07033066749572754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,10240,0.09081599712371827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,12288,0.10559893449147542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,16384,0.14007360140482586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,4096,0.04647253354390462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,5120,0.05500160058339437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,6144,0.061840001742045084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,3584,0.04165226618448893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,3072,0.03794133265813192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,2560,0.03400426705678304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,1536,0.02691733241081238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,2048,0.03086613416671753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,1024,0.022977066040039063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,65536,0.5712010701497395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,512,0.020337067047754922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,768,0.021312000354131062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,256,0.018506666024525963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,64,0.01806186636288961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,128,0.018203733364741008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,32,0.01776960094769796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,7168,0.06137706836064657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,8192,0.06897280216217042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,10240,0.08073066870371501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,16384,0.12138559818267822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,12288,0.09222506682078044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,3584,0.036722131570180255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,6144,0.054765868186950686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,3072,0.033718399206797284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,4096,0.040326400597890215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,5120,0.04792746702829997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,65536,0.5238047917683919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,2560,0.030734932422637938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,1024,0.020872533321380615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,1536,0.023928532997767128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,2048,0.02765120069185893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,256,0.01798293391863505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,768,0.020169599850972494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,512,0.01944640080134074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,128,0.01738986571629842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,64,0.01733120083808899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,32,0.017386666933695474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,8192,0.06420693397521973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,12288,0.0882250706354777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,7168,0.05843520164489746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,10240,0.07572267055511475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,16384,0.11298240025838216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,65536,0.45536425908406575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,6144,0.05224746863047282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,5120,0.04640746514002482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,4096,0.039001599947611494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,2560,0.029917865991592407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,3584,0.03579946756362915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,3072,0.033187200625737504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,768,0.020768000682195028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,1536,0.024820266167322795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,2048,0.027537065744400024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,1024,0.02101226647694906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,512,0.019458132982254028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,256,0.018500266472498576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,128,0.017831466595331826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,64,0.01821546753247579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,32,0.017899733781814576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,10240,0.07403199672698975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,7168,0.05516800085703531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,8192,0.06263573169708252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,12288,0.08474240303039551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,16384,0.10716053644816081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,6144,0.04999786615371704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,65536,0.42756160100301105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,4096,0.036526934305826826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,5120,0.04297279914220174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,3072,0.030932267506917317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,3584,0.03349440097808838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,2560,0.028479999303817748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,2048,0.02574933369954427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,1536,0.02339306672414144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,1024,0.02020906607309977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,256,0.017593600352605186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,768,0.019602133830388387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,512,0.01872533361117045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,128,0.01707093318303426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,64,0.017179733514785765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,32,0.01689066688219706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,10240,0.06766400337219239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,7168,0.050995198885599766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,12288,0.07743893464406332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,8192,0.0573365330696106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,16384,0.0979744036992391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,6144,0.045517865816752115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,65536,0.4052480061848958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,3584,0.03174399932225545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,5120,0.038925866285959884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,2560,0.02685653368631999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,4096,0.03423466682434082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,3072,0.029154133796691895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,768,0.018963199853897095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,2048,0.02445440093676249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,1536,0.022717867294947305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,1024,0.020055466890335084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,512,0.018437333901723228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,256,0.0170741339524587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,128,0.016833066940307617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,64,0.016729599237442015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,32,0.016845866044362386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,10240,0.07141760190327963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,12288,0.08211306730906168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,7168,0.04864319960276286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,8192,0.053704531987508144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,16384,0.10465493202209472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,65536,0.36566292444864906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,6144,0.04243733485539754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,5120,0.03737706740697225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,4096,0.03279146750768026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,3584,0.030162133773167926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,3072,0.02797866662343343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,2560,0.025652267535527545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,2048,0.023012266556421915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,1536,0.021181867520014445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,1024,0.01906879941622416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,128,0.016497066617012023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,256,0.016759467124938966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,512,0.017794134219487508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,768,0.018348799149195353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,64,0.0164000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,32,0.01639573375384013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,8192,0.04725866715113322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,16384,0.08467413584391276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,7168,0.0420138676961263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,10240,0.057550934950510654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,12288,0.06703466574350993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,65536,0.3399413426717123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,5120,0.033292800188064575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,4096,0.02994026740392049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,6144,0.03759466807047526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,3584,0.02754773298899333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,3072,0.025679999589920045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,2560,0.023985066016515098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,2048,0.021875200668970744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,1536,0.021272534132003786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,128,0.01633280018965403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,1024,0.019243733088175455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,256,0.016269866625467935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,512,0.017720532417297364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,768,0.017972266674041747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,32,0.015958399573961893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,64,0.01655359963575999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,8192,0.04256960153579712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,10240,0.05499626795450846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,12288,0.07029120127360025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,65536,0.31146666208902996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,16384,0.09126400152842204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,7168,0.03942826588948568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,6144,0.03585813442866008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,3584,0.02523840069770813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,2560,0.02281493345896403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,3072,0.024296534061431885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,5120,0.03186773260434468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,4096,0.028433066606521607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,2048,0.0210207998752594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,1536,0.019412267208099365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,1024,0.01800853411356608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,768,0.017310933272043864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,512,0.016823466618855795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,256,0.016062933206558227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,64,0.015893333156903586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,128,0.015770666797955833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,32,0.01590826710065206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,65536,0.276253859202067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,8192,0.039638400077819824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,12288,0.058310401439666745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,7168,0.036610134442647296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,10240,0.04807680050532023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,16384,0.07629760106404623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,3072,0.025195733706156416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,6144,0.03306879997253418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,3584,0.027159466346104937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,5120,0.03285973270734151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,2560,0.022804266214370726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,4096,0.028915200630823774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,2048,0.021516799926757812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,768,0.017498666048049928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,1536,0.01963520050048828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,1024,0.018169599771499633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,512,0.017148800690968833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,256,0.016127999623616537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,128,0.015622400244077048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,64,0.015595733126004537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,32,0.015474133690198264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,65536,0.2592127958933512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,8192,0.037758934497833255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,12288,0.054080001513163244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,7168,0.034585599104563394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,10240,0.04470826784769694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,16384,0.07302079995473226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,6144,0.031336534023284915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,3072,0.02304746707280477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,5120,0.028334933519363403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,3584,0.024460800488789878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,4096,0.024963200092315674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,2560,0.02132800022761027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,2048,0.020067199071248373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,1536,0.018703999121983846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,1024,0.01747093399365743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,768,0.01704533298810323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,512,0.016683733463287352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,256,0.01566506624221802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,128,0.01541759967803955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,64,0.01570026675860087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,32,0.015559466679890952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,65536,0.24757653872172036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,7168,0.03362773259480794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,8192,0.03588159879048665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,10240,0.04215466578801473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,12288,0.050169599056243894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,16384,0.0689791997273763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,6144,0.030510934193929036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,5120,0.027431466182072956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,4096,0.025095466772715253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,3584,0.023710934321085612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,3072,0.02267626722653707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,2560,0.02113386591275533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,2048,0.020119466384251914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,1536,0.018478933970133463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,1024,0.01736533244450887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,768,0.016829866170883178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,512,0.016216533382733662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,256,0.01564586659272512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,128,0.015631999572118124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,64,0.01520746648311615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,32,0.015347199638684592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,65536,0.2342591921488444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,7168,0.03157866597175598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,8192,0.03574613332748413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,12288,0.04676906665166219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,10240,0.03994239966074626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,6144,0.029849600791931153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,16384,0.06379199822743734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,5120,0.0272597332795461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,4096,0.02485226591428121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,2560,0.020932267109553017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,3584,0.023570134242375692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,3072,0.022375466426213582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,2048,0.01952000061670939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,1024,0.017086933056513466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,1536,0.018427733580271402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,768,0.016902399063110352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,256,0.01566506624221802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,512,0.016245333353678386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,128,0.015390933553377787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,64,0.01520639955997467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,32,0.015219199657440185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,65536,0.22676053047180175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,8192,0.03373759984970093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,7168,0.03192746639251709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,12288,0.04461439847946167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,10240,0.03918399810791016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,16384,0.06240959962209066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,6144,0.029768532514572142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,4096,0.024778666098912557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,5120,0.027447466055552167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,2048,0.019499733050664266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,2560,0.021031467119852702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,3072,0.022036266326904298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,3584,0.023371734221776328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,1536,0.01829119920730591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,1024,0.01709973414738973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,768,0.016643200318018594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,512,0.016489600141843162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,256,0.015221333503723145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,128,0.01548693378766378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,64,0.015260799725850423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,32,0.015107199549674988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,65536,0.22467734018961588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,7168,0.03174186746279399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,10240,0.038101331392923994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,12288,0.04362346728642781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,8192,0.033460267384847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,16384,0.06046186685562134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,6144,0.029838933547337847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,5120,0.02728319962819417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,3072,0.022194133202234904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,4096,0.024689066410064697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,2048,0.019615999857584634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,2560,0.020709333817164104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,3584,0.023027199506759643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,1024,0.016965333620707193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,1536,0.018388267358144125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,512,0.01632213294506073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,768,0.016702934106191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,256,0.0155349334081014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,128,0.015313067038853965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,64,0.015313067038853965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,32,0.014989866813023885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,65536,0.22349440256754555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,5120,0.24824426968892416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,6144,0.2962186813354492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,7168,0.347269344329834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,8192,0.3830005327860514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,10240,0.4721973419189453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,3584,0.18435413042704266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,3072,0.16433812777201334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,4096,0.20450453758239745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,12288,0.5677471796671549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,2560,0.1399722735087077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,1024,0.07704426447550455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,768,0.0665183981259664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,1536,0.09804480075836182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,256,0.04980053504308064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,2048,0.11999359925587971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,512,0.062062935034434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,128,0.04546026786168416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,64,0.04573653141657512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,32,0.047967998186747234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,16384,0.7439210891723633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,7168,0.10308372974395752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,8192,0.11352427005767822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,10240,0.14046613375345868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,12288,0.16349652608235676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,6144,0.09538773695627847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,5120,0.07887252966562906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,4096,0.06607573429743449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,3584,0.061324799060821535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,16384,0.21049493153889975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,3072,0.05520426829655965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,1024,0.030856533845265703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,2560,0.04874879916508992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,1536,0.036645332972208663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,768,0.028445865710576373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,2048,0.042889599005381265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,256,0.022874667247136434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,512,0.026126933097839356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,128,0.02118720014890035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,64,0.02057066758473714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,32,0.021476266781489055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,7168,0.08606399695078532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,8192,0.09442666371663412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,10240,0.11531733671824138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,12288,0.1358250617980957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,16384,0.1761365254720052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,5120,0.06530026594797769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,6144,0.07626880009969075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,3072,0.0469653328259786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,4096,0.05658560196558634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,3584,0.05175146659215292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,1536,0.03198506633440654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,1024,0.027160533269246417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,2560,0.04137066602706909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,2048,0.0370197335879008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,768,0.025201066335042314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,512,0.02305813431739807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,256,0.020963199933369956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,128,0.019045333067576088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,64,0.019802665710449217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,32,0.01988160014152527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,8192,0.08521386782328287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,10240,0.1021877368291219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,12288,0.11778879960378011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,65536,0.8231562932332357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,16384,0.15055893262227377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,6144,0.06886080106099447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,5120,0.05930453141530355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,7168,0.07738239765167236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,3584,0.04766720136006673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,4096,0.05527253150939941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,2560,0.03882026672363281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,3072,0.04350613355636597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,2048,0.03411519924799601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,1536,0.02964479923248291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,65536,0.6782325108846028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,1024,0.025844266017278034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,64,0.01878933310508728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,768,0.02390506664911906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,512,0.021993599335352578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,128,0.01889386574427287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,256,0.02090559999148051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,32,0.019223467508951823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,7168,0.06504319906234741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,8192,0.07208426793416342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,10240,0.08570880095163981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,12288,0.09926400184631348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,16384,0.12375893592834472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,3584,0.04090133508046468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,4096,0.04411093393961589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,5120,0.050860798358917235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,6144,0.05797226826349894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,3072,0.036602667967478436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,65536,0.5784255981445312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,2560,0.03392213185628255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,2048,0.029735465844472248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,512,0.020105600357055664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,768,0.022052266200383506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,1536,0.02658560077349345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,1024,0.023527467250823976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,64,0.01826559901237488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,128,0.017997866868972777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,256,0.018901334206263224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,32,0.018093866109848023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,7168,0.06205653349558512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,8192,0.0694432020187378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,12288,0.09609706401824951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,10240,0.08215359846750894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,16384,0.12115733623504639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,4096,0.04123306671778361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,3584,0.03769386609395345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,5120,0.049864534536997476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,6144,0.056068265438079835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,2560,0.030844799677530926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,3072,0.03408853212992351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,65536,0.47822933197021483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,2048,0.02749333381652832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,768,0.020593067010243736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,1024,0.02144320011138916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,1536,0.024679466088612875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,512,0.019827200969060262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,64,0.017997866868972777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,128,0.017782400051752724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,256,0.017356799046198527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,32,0.018154666821161906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,8192,0.06087573369344076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,7168,0.05522666772206625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,10240,0.07325226465861003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,12288,0.08484799861907959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,16384,0.1066805362701416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,5120,0.044227198759714766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,6144,0.05001706679662069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,3584,0.03533973296483357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,4096,0.038737066586812335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,3072,0.032332799832026166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,65536,0.45585066477457686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,1536,0.024434133370717367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,2048,0.026769065856933595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,1024,0.02179093360900879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,2560,0.0292138675848643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,256,0.017821866273880004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,768,0.020564266045888267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,512,0.018476800123850504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,128,0.017118932803471883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,64,0.017435733477274576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,32,0.017356799046198527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,10240,0.06716053485870362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,7168,0.05176746845245361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,8192,0.05733760197957357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,12288,0.07785600026448568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,6144,0.04688640038172404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,16384,0.09742613633473715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,5120,0.041637333234151204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,4096,0.03605440060297648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,3072,0.030701865752538044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,3584,0.033292800188064575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,65536,0.39812374114990234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,2560,0.028304000695546467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,1536,0.023890133698781332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,2048,0.026115200916926068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,1024,0.020680532852808634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,512,0.01923840045928955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,768,0.02038080096244812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,32,0.017361066738764443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,128,0.01741546591122945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,256,0.018252799908320107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,64,0.017270400126775106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,7168,0.04597333272298177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,12288,0.0684063990910848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,8192,0.05080639918645223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,10240,0.05982186794281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,16384,0.08538026809692383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,6144,0.041493332386016844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,5120,0.03619946638743083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,65536,0.3766677220662435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,3584,0.029497599601745604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,4096,0.03160213430722554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,2048,0.02361066738764445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,3072,0.027481599648793535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,2560,0.025552000602086383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,1536,0.021670399109522502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,1024,0.019700266917546592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,512,0.01803626616795858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,768,0.018850133816401164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,128,0.016569599509239197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,256,0.017138133446375527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,64,0.016696532567342125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,32,0.01676373283068339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,8192,0.04771093527475993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,7168,0.04292053381601969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,12288,0.06340906620025635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,10240,0.05580586592356364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,6144,0.03803413311640422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,16384,0.07914346853892008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,65536,0.30995200475056967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,5120,0.0339957316716512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,3072,0.025961599747339886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,4096,0.030152533451716108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,3584,0.02834239999453227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,2560,0.0246453324953715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,1024,0.019194666544596353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,2048,0.022997333606084188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,1536,0.020171733697255452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,768,0.01812480092048645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,128,0.01641493340333303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,512,0.017831466595331826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,256,0.016808533668518068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,32,0.016586666305859886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,64,0.016593066851298015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,8192,0.04462080001831055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,7168,0.040142933527628585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,10240,0.05658453305562337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,12288,0.06124693155288696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,16384,0.07543466885884603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,65536,0.288376522064209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,5120,0.032026666402816775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,6144,0.03603839874267578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,3072,0.025212800502777098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,4096,0.028862933317820232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,2560,0.02370026707649231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,3584,0.026842667659123735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,1536,0.019770665963490804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,2048,0.02206719915072123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,1024,0.018776534001032512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,512,0.017499732971191406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,768,0.018132267395655315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,256,0.016695467631022136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,32,0.016355199615160625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,128,0.016173866391181946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,64,0.016343466440836587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,10240,0.054790401458740236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,12288,0.062377599875132236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,7168,0.03923199971516927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,8192,0.04151893456776937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,65536,0.28132266998291017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,16384,0.07289386590321859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,6144,0.035979731877644854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,3584,0.026420267422993977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,4096,0.02834986646970113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,5120,0.03196266690889994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,2048,0.021126399437586464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,3072,0.02474453250567118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,2560,0.022740266720453896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,512,0.01722453236579895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,1536,0.019809067249298096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,1024,0.01837973395983378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,768,0.017835734287897746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,256,0.016316800316174825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,64,0.016108799974123636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,128,0.016134400169054666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,32,0.016127999623616537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,65536,0.2630741278330485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,12288,0.05931946833928427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,10240,0.05184106826782227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,8192,0.040880000591278075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,7168,0.037282133102416994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,16384,0.06871466636657715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,4096,0.027164800961812334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,6144,0.0343285322189331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,5120,0.030126933256785077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,3072,0.023900800943374635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,3584,0.02587839961051941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,2560,0.02178666591644287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,2048,0.020512000719706217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,1536,0.019411200284957887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,1024,0.017731199661890663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,768,0.01740586757659912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,512,0.016782933473587038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,128,0.015787733594576518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,256,0.016310399770736693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,64,0.015799466768900552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,32,0.01576746702194214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,65536,0.233842134475708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,12288,0.05612266858418783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,16384,0.06831040382385253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,8192,0.03864959875742595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,10240,0.0478986660639445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,7168,0.03513280153274536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,3584,0.02476373314857483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,6144,0.032689066727956136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,5120,0.029396265745162964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,2560,0.020805333058039346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,4096,0.026599466800689697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,3072,0.023524266481399537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,2048,0.02004693349202474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,1024,0.01744426687558492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,1536,0.018861865997314452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,768,0.017076265811920167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,512,0.01648319959640503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,256,0.01611733337243398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,128,0.015723733107248943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,64,0.015666133165359496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,32,0.015738667050997416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,65536,0.21323199272155763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,8192,0.03512533505757649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,12288,0.048410665988922116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,10240,0.0423093318939209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,16384,0.06201493342717489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,7168,0.03248639901479085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,6144,0.030084266265233355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,5120,0.027075199286142985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,3584,0.02262186606725057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,4096,0.023477333784103393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,2560,0.02018773357073466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,3072,0.021589332818984987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,2048,0.019062399864196777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,1024,0.017146666844685875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,1536,0.017948800325393678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,768,0.01680533289909363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,512,0.016501333316167197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,128,0.015637333194414772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,256,0.01574399967988332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,64,0.0156768004099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,32,0.01569919983545939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,65536,0.1920095920562744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,8192,0.03256426652272542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,7168,0.02977919975916545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,12288,0.04196159839630127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,10240,0.036858665943145755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,6144,0.028706133365631104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,16384,0.055801598230997715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,4096,0.024243199825286867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,5120,0.025512532393137617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,2560,0.020201599597930907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,3584,0.022443733612696328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,2048,0.019330133994420372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,3072,0.02110613385836283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,1536,0.01831573247909546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,1024,0.017171200116475424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,768,0.016597333550453185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,512,0.016473600268363954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,256,0.015768532951672874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,128,0.015410133202870688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,64,0.015441067020098367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,32,0.015348266561826071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,65536,0.18322240511576335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,12288,0.04206613302230835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,16384,0.05395306746164957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,7168,0.029586132367451983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,8192,0.031853866577148435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,10240,0.03761279980341593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,6144,0.027347199122111004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,3584,0.022740266720453896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,5120,0.02515733242034912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,3072,0.02117439905802409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,4096,0.023306665817896526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,2560,0.019666133324305217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,2048,0.01880533297856649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,1536,0.017757866779963175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,768,0.016471466422080992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,1024,0.017108267545700072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,512,0.01612053314844767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,256,0.01546986699104309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,128,0.015290666619936624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,65536,0.17395307223002116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,64,0.015521066387494406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,32,0.015268266201019287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,16384,0.04557120005289714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,8192,0.03035093347231547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,6144,0.026667733987172444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,7168,0.028743465741475422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,10240,0.03366400003433227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,12288,0.03743360042572021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,3072,0.020396800835927327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,3584,0.02140373388926188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,4096,0.022474666436513267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,5120,0.024232532580693564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,2048,0.018428800503412883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,2560,0.019359999895095827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,1536,0.01763946612675985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,1024,0.016703999042510985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,768,0.0161461333433787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,512,0.016035200158754984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,65536,0.16327039400736493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,256,0.015326933066050211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,128,0.015619200468063355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,64,0.015411200126012168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,32,0.01524906655152639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,8192,0.02943893273671468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,12288,0.03654826482137044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,10240,0.03279146750768026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,16384,0.04236693382263183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,7168,0.027988266944885255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,6144,0.026502400636672974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,2560,0.019352533419926963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,2048,0.01853013237317403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,4096,0.022460800409317017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,3072,0.020336000124613444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,5120,0.024258132775624594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,3584,0.021314134200414024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,1024,0.01665600041548411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,1536,0.017624533176422118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,512,0.016122666994730632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,768,0.016344533363978068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,256,0.015636266271273295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,65536,0.15861973762512208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,128,0.015227733055750528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,64,0.015310933192571005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,32,0.01525973379611969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,10240,0.031204267342885332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,8192,0.028388265768686933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,16384,0.041093333562215166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,6144,0.025973333915074663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,7168,0.027635200818379717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,12288,0.033521068096160886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,2560,0.019528534015019736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,3072,0.020102399587631225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,5120,0.024114133914311726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,4096,0.02241386572519938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,3584,0.02127679983774821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,2048,0.018346667289733887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,1024,0.01638826628526052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,1536,0.017308799425760905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,768,0.01616426706314087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,256,0.015387733777364096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,512,0.01583999991416931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,65536,0.15733973185221356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,128,0.015270400047302245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,64,0.015170133113861084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,32,0.015081600348154704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,10240,0.03126186728477478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,8192,0.028666667143503827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,6144,0.02638186613718669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,16384,0.040003200372060135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,12288,0.03288000027338664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,7168,0.02763413389523824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,3584,0.02121280034383138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,5120,0.024168533086776734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,2048,0.018218666315078735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,4096,0.02219946583112081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,3072,0.020292266209920248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,2560,0.019029333194096883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,768,0.0162090669075648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,1536,0.017395200332005818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,1024,0.016734933853149413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,256,0.015362133582433065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,512,0.015955199797948204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,65536,0.1547050635019938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,128,0.015254400173823037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,64,0.015085867047309876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,32,0.015040000279744467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,5120,0.20324479738871254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,6144,0.23502079645792642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,7168,0.2683616002400716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,8192,0.29928852717081705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,10240,0.37422078450520835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,4096,0.16768959363301594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,3584,0.1499498685201009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,3072,0.13244907061258954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,2560,0.11518613497416179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,1024,0.06862506866455079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,12288,0.43774932225545243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,1536,0.08281493186950684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,256,0.04400853315989177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,2048,0.0981877326965332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,512,0.05184853474299113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,128,0.04185280005137126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,768,0.060762667655944826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,32,0.04301973183949788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,64,0.03974719842274983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,16384,0.5824949264526367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,7168,0.08392639954884848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,8192,0.09332160154978433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,10240,0.11151680151621501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,12288,0.12946986357371013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,6144,0.07878719965616862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,4096,0.05471146504084269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,16384,0.1664138634999593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,5120,0.06451093355814616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,3584,0.05019946495691935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,3072,0.04583573341369629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,1024,0.02723306616147359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,1536,0.03165653347969055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,2560,0.04133653243382772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,2048,0.03657279809315999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,256,0.021109332640965782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,768,0.025384533405303954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,512,0.023414399226506552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,128,0.019461333751678467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,32,0.019859200716018675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,64,0.0197760005791982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,8192,0.07452159722646078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,7168,0.06737279891967773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,10240,0.08901440302530925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,12288,0.10351040363311767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,16384,0.13291412989298504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,6144,0.06005119880040487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,5120,0.052775466442108156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,4096,0.04541973272959392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,2560,0.035259731610616046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,3584,0.04250880082448323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,2048,0.031204267342885332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,1024,0.024679466088612875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,3072,0.038894931475321456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,1536,0.027888000011444092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,768,0.02330986658732096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,256,0.01927893360455831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,512,0.02130026618639628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,128,0.018437333901723228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,64,0.018522665898005167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,32,0.019035732746124266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,8192,0.06905492941538492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,10240,0.08220693270365396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,12288,0.09603520234425864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,65536,0.6469856262207031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,16384,0.12187413374582927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,7168,0.06295359929402669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,6144,0.056187733014424646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,3584,0.040755200386047366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,5120,0.050086398919423424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,4096,0.046562135219573975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,2560,0.033089067538579306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,3072,0.03712639808654785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,2048,0.029956267277399702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,1536,0.026228266954421996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,65536,0.502348804473877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,1024,0.02369706630706787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,512,0.020563199122746786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,768,0.0221834659576416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,64,0.017795199155807497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,256,0.01862506667772929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,128,0.01811199982961019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,32,0.018204800287882485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,7168,0.05334826707839966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,10240,0.06978346506754557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,12288,0.08095680077870687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,8192,0.058259201049804685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,16384,0.10040853023529053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,4096,0.03705919981002807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,3584,0.03407040039698283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,5120,0.04284480015436808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,3072,0.031141332785288495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,6144,0.0482965350151062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,65536,0.44647998809814454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,2560,0.028862933317820232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,2048,0.02618880073229472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,1536,0.02433919906616211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,1024,0.02170026699701945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,256,0.018129066626230875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,512,0.018696532646814982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,768,0.02074346741040548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,128,0.017285333077112833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,64,0.017513600985209148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,32,0.01746666630109151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,7168,0.05139093399047852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,12288,0.0766869306564331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,8192,0.056060798962910975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,16384,0.09550399780273437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,10240,0.066484268506368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,3584,0.03315839966138204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,4096,0.03618133465449015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,3072,0.030451200405756634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,2560,0.028283733129501342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,6144,0.04608533382415771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,5120,0.04157013495763143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,2048,0.025880533456802367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,65536,0.36755520502726235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,1024,0.021844265858332317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,1536,0.023656533161799113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,768,0.019564799467722573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,512,0.019273600975672402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,64,0.017858133713404337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,256,0.01843520005544027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,128,0.017692800362904867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,32,0.017914666732152303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,12288,0.07075733343760172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,8192,0.05282986561457316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,10240,0.06332266728083293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,7168,0.049219199021657306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,16384,0.08881386915842691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,4096,0.03418773412704468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,5120,0.040085331598917646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,65536,0.35402345657348633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,6144,0.04477013349533081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,3584,0.03173440098762512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,3072,0.029808000723520918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,1024,0.02070186734199524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,2560,0.027509333690007527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,2048,0.024819199244181314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,1536,0.022755199670791627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,768,0.018423465887705485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,512,0.018338133891423546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,256,0.017476266622543334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,128,0.01689173380533854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,32,0.017062399784723917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,64,0.017025067408879598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,8192,0.048148266474405926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,7168,0.0440341313680013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,12288,0.06443093220392862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,10240,0.056377601623535153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,16384,0.0806336005528768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,4096,0.030425600210825604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,65536,0.317464542388916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,6144,0.03970346848169963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,5120,0.03499626715977987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,3584,0.02912000020345052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,3072,0.026808534065882367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,2560,0.0248799999554952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,1536,0.0214303990205129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,1024,0.01915093262990316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,2048,0.022974934180577597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,768,0.01839146614074707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,256,0.017143466075261436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,512,0.017774933576583864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,128,0.01679253379503886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,64,0.016775466998418174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,32,0.016667733589808144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,7168,0.03996479908625285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,10240,0.059463465213775636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,8192,0.043661868572235106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,16384,0.08099839687347413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,12288,0.0680021365483602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,65536,0.27787307103474934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,6144,0.03617493311564128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,5120,0.03168213367462158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,3584,0.026446932554244997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,4096,0.02872640093167623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,3072,0.02510506709416707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,2048,0.021553067366282146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,1024,0.01816640098889669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,2560,0.023504000902175904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,1536,0.020619734128316244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,768,0.017953066031138103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,512,0.01728106737136841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,128,0.016313599546750386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,256,0.016408532857894897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,32,0.016429866353670754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,64,0.01620693306128184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,8192,0.04251733223597209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,7168,0.03839253187179566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,65536,0.2518335978190104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,10240,0.04949333270390828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,12288,0.05645866791407267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,16384,0.07028586864471435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,6144,0.03502080043156942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,4096,0.027885866165161134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,5120,0.030573866764704388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,3072,0.024734934171040855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,3584,0.026013867060343428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,2048,0.021619200706481934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,2560,0.02327893376350403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,1536,0.019809067249298096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,768,0.017735467354456583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,1024,0.018337066968282065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,512,0.017233065764109292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,256,0.016743467251459757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,128,0.016420267025629678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,32,0.016361600160598753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,64,0.016518400112787882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,8192,0.03966079950332642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,7168,0.035786668459574386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,65536,0.23756373723347984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,10240,0.046431998411814376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,12288,0.0528223991394043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,16384,0.06456106503804525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,6144,0.03275093237559001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,3072,0.02354773283004761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,5120,0.02914773424466451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,4096,0.02646506627400716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,2560,0.021744000911712646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,3584,0.02536746660868327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,1536,0.019368533293406168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,2048,0.0207914670308431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,1024,0.018011732896169027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,512,0.016804265975952148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,768,0.01760960022608439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,256,0.016421332955360413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,128,0.016194132963816325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,64,0.016217600305875143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,32,0.01606186628341675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,8192,0.036735999584198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,10240,0.0434442679087321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,7168,0.03294186592102051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,12288,0.050088532765706385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,65536,0.22605652809143068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,16384,0.061520000298817955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,6144,0.030798933903376263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,5120,0.02741439938545227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,4096,0.025040000677108765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,3584,0.023871999979019166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,2560,0.021268266439437866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,3072,0.022626133759816487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,2048,0.019156267245610557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,1536,0.018716800212860107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,1024,0.017607466379801432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,512,0.016664533813794454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,768,0.017121066649754844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,256,0.016343466440836587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,128,0.01604693333307902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,64,0.015918933351834617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,32,0.01592639982700348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,65536,0.20474559466044107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,10240,0.04256853262583415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,8192,0.03482453425725301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,12288,0.05200426578521729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,7168,0.032048000892003374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,16384,0.06634560028711954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,6144,0.029789867003758748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,5120,0.027349332968393963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,4096,0.02445546587308248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,3072,0.021381332476933797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,3584,0.02330986658732096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,2560,0.020113066832224528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,1536,0.018453333775202432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,1024,0.01730453372001648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,2048,0.01895786722501119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,768,0.01713706652323405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,512,0.01629866659641266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,64,0.015713066856066386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,128,0.0156960000594457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,256,0.016217600305875143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,32,0.01569706698258718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,65536,0.1840117295583089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,10240,0.03744746843973796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,7168,0.029689600070317585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,12288,0.04309759934743245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,8192,0.03259946703910828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,16384,0.054364800453186035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,6144,0.027795199553171796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,5120,0.02718399961789449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,3584,0.023038933674494423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,4096,0.024605866273244223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,3072,0.021922133366266885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,2560,0.019656533002853395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,2048,0.01943040092786153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,1536,0.018147200345993042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,1024,0.01737066706021627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,768,0.016663466890652977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,512,0.016264533003171287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,128,0.015521066387494406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,256,0.01611733337243398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,64,0.015502933661142984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,32,0.01551253298918406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,65536,0.18025066057840983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,7168,0.027731200059254963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,6144,0.02616746624310811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,8192,0.02997973362604777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,12288,0.03851946592330933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,16384,0.049398398399353026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,10240,0.03373333215713501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,5120,0.024899200598398844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,3584,0.021588265895843506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,4096,0.02298240065574646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,3072,0.02063573400179545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,2560,0.019850667317708334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,2048,0.019058134158452353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,1536,0.018157867590586345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,1024,0.01703146696090698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,768,0.016748799880345663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,512,0.016260266304016113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,65536,0.1590976079305013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,256,0.015733333428700765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,128,0.015383467078208923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,64,0.015450666348139444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,32,0.01544533371925354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,6144,0.02566186587015788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,12288,0.037254401048024494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,7168,0.02753173311551412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,10240,0.0335370659828186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,16384,0.04710719982783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,8192,0.029709867636362713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,5120,0.023715200026830037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,3584,0.020882133642832437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,4096,0.021514666080474854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,2560,0.019112533330917357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,3072,0.020221867163976035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,2048,0.018331732352574667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,1536,0.017746132612228394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,1024,0.016801067193349204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,512,0.01620693306128184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,65536,0.145633061726888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,768,0.016328533490498863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,256,0.015878400206565856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,128,0.015367466211318969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,64,0.015331199765205384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,32,0.01528320014476776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,12288,0.03540266752243042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,8192,0.02834239999453227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,10240,0.03217173417409261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,16384,0.04297813177108765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,7168,0.026331732670466106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,6144,0.024567466974258424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,5120,0.02335253357887268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,2560,0.018795732657114664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,2048,0.018310399850209554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,4096,0.021625600258509316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,3584,0.02081600030263265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,3072,0.019973333676656088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,1024,0.01660053332646688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,768,0.016410666704177856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,1536,0.017476266622543334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,128,0.015243732929229736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,65536,0.1378346602121989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,256,0.015749333302179973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,512,0.01569706698258718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,64,0.015149866541226705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,32,0.015254400173823037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,12288,0.033787735303243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,10240,0.028755199909210206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,16384,0.038320000966389975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,7168,0.025922133525212603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,6144,0.024759467442830405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,8192,0.02797120014826457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,3584,0.02012373407681783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,5120,0.022762666145960488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,2560,0.018515199422836304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,3072,0.01936639944712321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,4096,0.021195733547210695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,2048,0.017895466089248656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,65536,0.12853333155314128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,512,0.015825066963831583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,1536,0.01741866668065389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,1024,0.016609066724777223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,768,0.01616853376229604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,256,0.01553813318411509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,128,0.015156267086664834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,64,0.01504853367805481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,32,0.015108266472816467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,12288,0.030239999294281006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,10240,0.028724267085393267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,6144,0.02451840043067932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,16384,0.036134401957194015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,7168,0.02550293405850728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,8192,0.026128000020980834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,3584,0.0200981338818868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,5120,0.022603732347488404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,4096,0.021280000607172646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,3072,0.019332265853881835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,2560,0.018282665808995565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,2048,0.01772800087928772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,65536,0.12530133724212647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,1536,0.017101866006851197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,1024,0.016425599654515587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,768,0.016134400169054666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,512,0.01585706671079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,256,0.015293866395950317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,32,0.015044266978899637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,128,0.015165866414705912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,64,0.015132799744606018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,8192,0.026129066944122314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,10240,0.028119466702143353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,7168,0.025282132625579833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,12288,0.029715200265248615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,6144,0.024529065688451132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,16384,0.03462506532669067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,5120,0.022605866193771362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,2560,0.018439465761184694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,4096,0.020907733837763467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,3072,0.019131733973821004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,3584,0.019973333676656088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,1536,0.01712426741917928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,2048,0.017885865767796834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,65536,0.12368853092193603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,128,0.015017599860827128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,1024,0.016456533471743265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,512,0.01575573285420736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,768,0.01609386702378591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,256,0.015520000457763672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,32,0.015118933717409768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,64,0.015150933464368185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,7168,0.025552000602086383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,10240,0.028310400247573853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,12288,0.029796266555786134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,16384,0.03368639945983887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,8192,0.02619306643803914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,6144,0.024394667148590087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,5120,0.022376533349355063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,4096,0.020804266134897866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,3584,0.020411733786265054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,2560,0.01835839947064718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,3072,0.01907520095507304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,1536,0.01726079980532328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,2048,0.01783039967219035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,65536,0.12070506413777668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,1024,0.016471466422080992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,128,0.015172266960144043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,768,0.016229333480199178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,512,0.015453867117563882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,256,0.015385599931081137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,64,0.015144532918930054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,32,0.014934399724006652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,5120,0.15888959566752117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,6144,0.18484692573547362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,7168,0.20846080780029297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,8192,0.23165440559387207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,4096,0.13340266545613605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,10240,0.2803733189900716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,3584,0.12164159615834554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,3072,0.10790826479593914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,12288,0.33613227208455404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,2560,0.09414400259653727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,2048,0.07981440226236979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,1536,0.06751466592152913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,768,0.047984000047047934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,256,0.03561280171076457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,512,0.04593813419342041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,1024,0.05460266669591268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,128,0.030956800778706866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,64,0.02985386649767558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,32,0.03221333424250285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,16384,0.4258303960164388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,7168,0.06910400390625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,10240,0.09058453241984049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,8192,0.07622933387756348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,12288,0.10465386708577473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,16384,0.1307904005050659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,6144,0.06715733210245768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,4096,0.04728000164031983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,3584,0.044061867396036784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,5120,0.05465813477834066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,3072,0.04052373170852661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,1536,0.027727999289830524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,2560,0.036373333136240645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,2048,0.031896533568700154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,768,0.022686932484308878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,1024,0.023749333620071412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,512,0.02142933408419291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,256,0.018248534202575682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,128,0.017770665884017944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,32,0.018685867389043175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,64,0.01816320021947225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,7168,0.056601599852244056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,10240,0.07405866781870524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,8192,0.06265279849370321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,12288,0.08561387062072753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,16384,0.10674880345662434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,3584,0.03743893305460612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,4096,0.03991039991378784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,5120,0.04547199805577596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,6144,0.051276799043019614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,2560,0.030871466795603437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,3072,0.034373335043589276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,1536,0.02453440030415853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,2048,0.027155200640360515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,1024,0.021977599461873373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,256,0.017654399077097573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,768,0.02089280088742574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,128,0.017219199736913045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,512,0.019028266270955406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,64,0.017267199357350667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,32,0.01793173352877299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,8192,0.056493866443634036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,10240,0.06657280127207438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,12288,0.07727999687194824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,65536,0.4579029401143392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,16384,0.09526933034261068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,7168,0.05166399876276652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,6144,0.04694933493932088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,5120,0.04181439876556396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,3072,0.03139946659406026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,4096,0.039239466190338135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,3584,0.03422826528549194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,1536,0.023792000611623128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,2560,0.02853013277053833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,768,0.02092906634012858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,1024,0.022175999482472737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,2048,0.025831466913223265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,512,0.01881386637687683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,65536,0.3598911921183268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,256,0.018100267648696898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,128,0.017114667097727458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,64,0.017443199952443443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,32,0.017534933487574258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,7168,0.045287466049194335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,8192,0.049201067288716635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,10240,0.057526401678721105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,12288,0.06608320077260335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,16384,0.08429973125457764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,3584,0.029840000470479328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,65536,0.35265493392944336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,4096,0.03221120039621989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,3072,0.02754773298899333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,5120,0.0369045337041219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,6144,0.041357866923014325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,2048,0.024488532543182374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,2560,0.025378133853276568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,1536,0.02178773283958435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,768,0.0181877334912618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,1024,0.02018453280131022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,512,0.017387733856836955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,256,0.016950400670369466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,128,0.016357333461443583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,64,0.01686933239301046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,32,0.017240534226099648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,12288,0.061244801680246985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,7168,0.041716265678405764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,8192,0.04548480113347371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,10240,0.05292799870173136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,16384,0.07581226825714112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,4096,0.03039466738700867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,65536,0.2699007987976074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,6144,0.03831466833750407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,3584,0.02763200004895528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,5120,0.034434131781260174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,3072,0.025730133056640625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,1536,0.02122453252474467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,2560,0.02434026598930359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,2048,0.023180800676345825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,1024,0.019525333245595296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,768,0.018232532342274985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,512,0.017722666263580322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,256,0.016899200280507405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,64,0.016595199704170227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,128,0.01636373301347097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,32,0.01660586694876353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,10240,0.04972586631774902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,7168,0.03914986848831177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,8192,0.04246826569239299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,12288,0.056932266553243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,16384,0.06997333367665609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,3584,0.026103466749191284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,4096,0.02866026759147644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,5120,0.031850665807724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,65536,0.24268159866333008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,6144,0.03598613341649373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,2560,0.023404800891876222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,3072,0.024778666098912557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,2048,0.021758933862050377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,1536,0.020163200298945107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,1024,0.01853440006573995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,512,0.017296000321706136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,768,0.017962666352589925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,256,0.016684800386428833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,128,0.016109866897265117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,64,0.016141866644223533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,32,0.016732800006866454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,16384,0.06459306478500366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,8192,0.03981226682662964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,7168,0.03720426559448242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,10240,0.0482965350151062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,12288,0.052813867727915444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,65536,0.23493332862854005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,5120,0.030385067065556843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,3584,0.02517333428064982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,6144,0.03418240149815877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,4096,0.02725546757380168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,3072,0.024290132522583007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,2560,0.0228000005086263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,1024,0.0180949330329895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,2048,0.021074134111404418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,1536,0.01986453334490458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,768,0.017602133750915527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,512,0.017172267039616905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,256,0.016371200482050575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,128,0.016076800227165223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,64,0.015960533420244852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,32,0.016556800405184428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,65536,0.21321279207865396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,7168,0.03495786587397258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,16384,0.05933866500854492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,8192,0.03768213192621867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,12288,0.05081280072530111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,10240,0.045620266596476236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,6144,0.03162026604016622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,5120,0.02905600070953369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,4096,0.025831466913223265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,3584,0.024244266748428344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,2048,0.020718934138615926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,3072,0.02332586646080017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,2560,0.021899733940760294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,1536,0.019553067286809285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,512,0.01655893325805664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,768,0.016979199647903443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,1024,0.017767467101415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,256,0.016249600052833556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,128,0.01576959987481435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,64,0.01591679950555166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,32,0.016131200393040977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,7168,0.032755200068155924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,8192,0.035913598537445066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,65536,0.18584319750467937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,10240,0.04088533322016398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,12288,0.04649600187937419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,16384,0.056849066416422525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,6144,0.030039467414220172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,5120,0.02720853288968404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,4096,0.02490880091985067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,3584,0.023568000396092734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,3072,0.02251946727434794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,2048,0.02007253368695577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,2560,0.02135253349939982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,1536,0.017989333470662436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,768,0.016945066054662068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,1024,0.01745599905649821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,512,0.016403200229008992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,256,0.015980799992879234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,32,0.016042666633923848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,64,0.015808000167210897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,128,0.015737600127855935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,10240,0.0401472012201945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,8192,0.03412373463312785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,65536,0.1766858736673991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,7168,0.030906667311986286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,12288,0.04513386487960815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,16384,0.05450559854507446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,5120,0.026173865795135497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,6144,0.028333866596221925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,3072,0.022054400046666464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,4096,0.02373653252919515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,3584,0.022802132368087768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,1024,0.017083734273910522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,2048,0.019419733683268228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,1536,0.018102399508158364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,2560,0.020807466904322305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,768,0.01654293338457743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,512,0.016406400005022685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,256,0.01576746702194214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,128,0.015457066893577575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,64,0.01569066643714905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,32,0.015811199943224587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,65536,0.16379094123840332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,12288,0.04059946537017822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,8192,0.03169386585553487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,10240,0.03844053347905477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,16384,0.049697065353393556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,6144,0.027452800671259565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,7168,0.029630933205286664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,4096,0.023363200823465984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,5120,0.02518613338470459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,3584,0.022179200251897176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,2560,0.0191594660282135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,3072,0.021229867140452066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,2048,0.018615466356277467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,1536,0.01764586567878723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,1024,0.016964266697565712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,768,0.016536532839139303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,512,0.016150400042533875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,256,0.01583573321501414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,64,0.015506133437156677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,128,0.01536853313446045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,32,0.015505066514015198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,65536,0.15291840235392254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,7168,0.027738666534423827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,10240,0.03480639855066935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,6144,0.02586666742960612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,8192,0.03006400068600972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,12288,0.0397216002146403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,16384,0.04824533462524414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,5120,0.023900800943374635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,4096,0.022256000836690267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,3584,0.020947200059890748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,3072,0.020388267437616982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,2560,0.018268799781799315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,2048,0.017479467391967773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,1536,0.017218132813771568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,1024,0.016897066434224447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,768,0.016179200013478598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,128,0.015464533368746439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,512,0.015983999768892924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,256,0.015921066204706825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,65536,0.1375658671061198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,64,0.015587199727694193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,32,0.01588373382886251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,6144,0.024200532833735147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,7168,0.025906133651733398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,8192,0.027480532725652058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,10240,0.031128533681233722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,16384,0.04224426746368408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,12288,0.03378986517588298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,5120,0.022859734296798707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,4096,0.021179733673731486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,3584,0.020092799266179403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,2560,0.018885332345962524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,3072,0.019309866428375243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,2048,0.017893334229787193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,1024,0.016804265975952148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,1536,0.017528533935546875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,768,0.01639786660671234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,512,0.016057599584261575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,65536,0.12451732953389485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,128,0.015171200037002563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,256,0.015762133399645488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,64,0.015474133690198264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,32,0.015608533223470052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,12288,0.03373653491338094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,16384,0.040174933274586995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,10240,0.031700267394383745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,8192,0.026664533217748004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,7168,0.02478826642036438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,6144,0.023988266785939537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,2048,0.01806079943974813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,2560,0.018538665771484376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,5120,0.02186453342437744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,4096,0.020940800507863365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,3584,0.01936533252398173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,3072,0.01895466645558675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,1536,0.017097600301106772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,1024,0.016665599743525186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,65536,0.11175680160522461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,768,0.016151466965675355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,512,0.015871999661127727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,256,0.015493333339691162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,128,0.015396266182263692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,64,0.015268266201019287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,32,0.015291733543078103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,12288,0.03224106629689534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,16384,0.03621333440144857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,8192,0.02627840042114258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,10240,0.02881173292795817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,6144,0.022873600323994957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,7168,0.023857067028681435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,4096,0.019831466674804687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,5120,0.021916800737380983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,2560,0.018261333306630455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,2048,0.017672532796859743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,3584,0.01946773330370585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,3072,0.018835200866063436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,65536,0.10553279717763264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,768,0.015918933351834617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,512,0.01581013302008311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,1536,0.016886399189631144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,1024,0.016748799880345663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,256,0.015570132931073507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,128,0.015173332889874777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,64,0.015222400426864624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,32,0.015105066696802774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,12288,0.029428267478942872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,6144,0.022478934129079184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,16384,0.0333024005095164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,8192,0.025177599986394246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,10240,0.02821439901987712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,7168,0.024565333127975465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,4096,0.019836799303690592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,5120,0.02151040037473043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,3584,0.019066667556762694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,2048,0.01738026738166809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,3072,0.018538665771484376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,2560,0.017944532632827758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,1536,0.016923733552296958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,65536,0.09858559767405192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,1024,0.016511999567349753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,768,0.01585493286450704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,64,0.015095466375350952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,512,0.015655466914176942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,256,0.015522133310635886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,128,0.01535040040810903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,32,0.015209600329399109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,10240,0.026610134045283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,12288,0.028942932685216267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,6144,0.022749867041905722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,16384,0.030204800764719646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,7168,0.023772799968719484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,8192,0.024752000967661537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,3584,0.019217065970102944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,4096,0.01965973377227783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,5120,0.0210698664188385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,3072,0.01830186645189921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,65536,0.09158079624176026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,2560,0.017821866273880004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,2048,0.017244799931844076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,1536,0.017003732919692992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,1024,0.016502400239308677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,768,0.016035200158754984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,512,0.015507200360298156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,256,0.015548800428708395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,128,0.015335466464360556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,32,0.01544319987297058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,64,0.015218133727709452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,7168,0.023626667261123658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,12288,0.02595306634902954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,16384,0.028653866052627562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,10240,0.025605332851409913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,8192,0.02370133399963379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,6144,0.022317866484324135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,5120,0.021075199047724404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,4096,0.019476266702016194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,3584,0.019172267119089762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,3072,0.0184170663356781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,65536,0.09014080365498861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,2560,0.01783039967219035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,1536,0.016753067572911583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,2048,0.017296000321706136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,768,0.01595093309879303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,1024,0.016089600324630738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,512,0.015655466914176942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,256,0.015338666240374246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,64,0.015129599968592325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,128,0.015186132987340293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,32,0.014969600240389505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,12288,0.026250666379928587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,10240,0.025436800718307496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,8192,0.02382826606432597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,16384,0.028731733560562134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,7168,0.023447465896606446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,6144,0.022230400641759237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,3584,0.019139200448989868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,4096,0.01954453388849894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,5120,0.021305600802103676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,3072,0.01827626625696818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,65536,0.08584213256835938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,2560,0.017768534024556477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,1536,0.016839466492335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,2048,0.017157334089279174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,1024,0.016084266702334087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,768,0.015963733196258545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,512,0.015340800086657206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,256,0.015380266308784484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,64,0.015035733580589294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,128,0.015255467096964518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,32,0.014872533082962037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,8192,0.023836799462636313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,10240,0.025399466355641682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,12288,0.025889066855112712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,16384,0.02878933350245158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,7168,0.023387734095255533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,6144,0.022105600436528525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,5120,0.02085226575533549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,4096,0.019885865847269694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,3584,0.018785067399342856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,3072,0.01847040057182312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,65536,0.083897598584493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,1536,0.017013333241144814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,2560,0.017947733402252197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,2048,0.017385600010553996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,1024,0.01639573375384013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,768,0.015979733069737753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,256,0.015068800250689188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,512,0.015618133544921874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,128,0.015098667144775391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,64,0.015148799618085226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,32,0.014850133657455444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,5120,0.15075732866923014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,6144,0.1735637346903483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,7168,0.19661653836568196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,8192,0.21944212913513184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,10240,0.26658560434977213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,3072,0.10362880229949951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,4096,0.12752853234608968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,3584,0.11591893037160236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,2560,0.09075199762980143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,12288,0.31332267125447594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,2048,0.07769066492716471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,128,0.02858240008354187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,256,0.03319360017776489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,768,0.04800533453623454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,1536,0.06577920118967692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,1024,0.05395413239796957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,512,0.04274880091349284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,64,0.02836693326632182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,32,0.029234133164087933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,16384,0.4049642562866211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,8192,0.07075520356496176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,7168,0.06365439891815186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,10240,0.0836458683013916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,12288,0.0970848004023234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,6144,0.06287146806716919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,16384,0.1212831974029541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,5120,0.051381333669026696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,4096,0.0443391998608907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,3584,0.04155840078989665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,3072,0.03822933435440064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,1536,0.026579199234644572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,2560,0.0344320019086202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,1024,0.022741333643595377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,2048,0.030382933219273885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,768,0.021594667434692384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,256,0.017578667402267455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,512,0.020170666774113975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,32,0.01821119983990987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,64,0.01754666765530904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,128,0.017258665959040322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,7168,0.05385919809341431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,8192,0.05820906559626261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,10240,0.06852800051371256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,12288,0.07883413632710776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,16384,0.0981760025024414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,4096,0.03791146675745646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,6144,0.04862613280614217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,5120,0.04336320161819458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,3072,0.032778666416804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,3584,0.035454932848612467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,2560,0.029469867547353108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,1536,0.023415466149648033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,2048,0.02598080039024353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,1024,0.021193599700927733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,768,0.02020373344421387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,64,0.016911999384562174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,512,0.017927465836207072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,256,0.017017600933710735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,128,0.01683626572291056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,32,0.017359999815622966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,10240,0.06209386587142944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,65536,0.41959788004557297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,12288,0.07053439617156983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,8192,0.05299199819564819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,16384,0.08899839719136557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,3584,0.032576000690460204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,5120,0.03930026690165202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,4096,0.03487999836603801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,7168,0.04896533489227295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,6144,0.04795413414637248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,3072,0.02991466720898946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,2560,0.026894932985305785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,1536,0.02332373261451721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,1024,0.020901334285736085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,2048,0.025493333737055462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,768,0.01981546680132548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,65536,0.35767787297566733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,32,0.017398399114608765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,256,0.017113600174585977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,512,0.01765120029449463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,128,0.01652479966481527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,64,0.016898133357365928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,7168,0.0420085350672404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,8192,0.04556800127029419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,10240,0.05428693294525146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,12288,0.062332801024119054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,16384,0.07668053309122722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,3584,0.028293333450953168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,3072,0.02579306761423747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,4096,0.030280532439549764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,5120,0.03455679814020793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,65536,0.314629332224528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,6144,0.03831679821014404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,2048,0.022486400604248048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,2560,0.024423466126124064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,1536,0.02093013326327006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,768,0.017658666769663493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,1024,0.019333332777023315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,512,0.01698346734046936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,256,0.01667520006497701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,128,0.01626240015029907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,32,0.016771199305852254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,64,0.016242133577664693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,8192,0.042985598246256515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,7168,0.039589333534240725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,10240,0.05313706795374552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,12288,0.061050665378570554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,16384,0.07522559960683187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,3584,0.028068266312281293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,4096,0.028537599245707194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,6144,0.03650986750920614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,65536,0.2555775960286458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,5120,0.032778666416804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,3072,0.026341332991917925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,2560,0.024477867285410564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,2048,0.02243946592013041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,1536,0.021217066049575805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,768,0.017387733856836955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,1024,0.019151999553044637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,256,0.01693120002746582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,512,0.017378133535385133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,128,0.016338133811950685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,64,0.016505600015322367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,32,0.01655893325805664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,8192,0.041598931948343916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,10240,0.05146239995956421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,7168,0.038873600959777835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,12288,0.056032001972198486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,16384,0.0697312037150065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,65536,0.2426538626352946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,5120,0.03213546673456828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,4096,0.02877333362897237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,3072,0.02533866763114929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,6144,0.03618133465449015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,3584,0.02635626594225566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,1024,0.018284799655278523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,2560,0.023523199558258056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,1536,0.02021546761194865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,768,0.01750613252321879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,2048,0.021782400210698445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,512,0.01696746746699015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,256,0.01626240015029907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,64,0.01588586668173472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,32,0.016196266810099284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,128,0.016084266702334087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,8192,0.038224001725514725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,7168,0.035249066352844236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,12288,0.04971733490626017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,10240,0.043969066937764485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,16384,0.06122453212738037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,65536,0.20923840204874672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,6144,0.03237546682357788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,4096,0.026143999894460042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,3584,0.02441920042037964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,5120,0.029331199328104657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,2560,0.022437334060668945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,2048,0.02074986696243286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,3072,0.023483733336130776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,1536,0.0198634664217631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,1024,0.017962666352589925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,256,0.016288000345230102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,768,0.017190400759379068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,512,0.01672853430112203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,128,0.0156960000594457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,64,0.01581333378950755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,32,0.01576640009880066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,65536,0.1889365355173747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,8192,0.03339946667353312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,7168,0.030343466997146608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,10240,0.04064000050226847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,12288,0.04895893335342407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,16384,0.0598090648651123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,5120,0.026149332523345947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,6144,0.02890666723251343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,4096,0.023844265937805177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,3072,0.021919999519983926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,3584,0.022886399428049722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,2560,0.021248000860214233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,2048,0.019766400257746376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,1536,0.018778665860493978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,1024,0.017151999473571777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,128,0.015713066856066386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,768,0.01696853240331014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,512,0.016681599617004394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,256,0.016104533274968465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,64,0.015896532932917276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,32,0.015940266847610473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,65536,0.16396692593892415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,8192,0.03602026700973511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,7168,0.03236800034840902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,10240,0.040608000755310056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,6144,0.030014934142430623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,16384,0.054622932275136316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,12288,0.04406826496124268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,3072,0.02207466761271159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,5120,0.026910932858784993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,4096,0.024643200635910033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,3584,0.02307413419087728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,2560,0.020916267236073812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,2048,0.020071466763814293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,1536,0.01829013427098592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,1024,0.016956800222396852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,768,0.016633599996566772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,512,0.01657386620839437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,64,0.015809067090352378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,256,0.016056533654530844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,128,0.015508266290028891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,32,0.015923200050989787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,65536,0.15459520022074383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,10240,0.03669120073318481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,8192,0.031446399291356404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,6144,0.02900586724281311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,12288,0.04179840087890625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,7168,0.027246934175491334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,16384,0.05054826736450195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,5120,0.02610879937807719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,3584,0.0226090669631958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,4096,0.02358506719271342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,3072,0.021913599967956544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,2560,0.020567466815312706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,2048,0.01881600022315979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,1024,0.017207467555999757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,1536,0.018102399508158364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,512,0.016370133558909098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,768,0.016718933979670204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,256,0.01597866714000702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,64,0.01567466656366984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,128,0.01569706698258718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,32,0.015864533185958863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,65536,0.14241706530253093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,10240,0.032891732454299924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,12288,0.037240533034006755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,8192,0.02904213269551595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,7168,0.02752959926923116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,6144,0.025290666023890178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,16384,0.044810668627421064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,5120,0.023471999168395995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,4096,0.021840000152587892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,1536,0.017486933867136636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,3072,0.020744532346725464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,3584,0.0211136003335317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,2560,0.018900267283121743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,2048,0.017705599466959633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,768,0.016379732886950174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,1024,0.016780799627304076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,512,0.016149333119392394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,256,0.015718400478363037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,128,0.015264000495274863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,32,0.015451733271280924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,64,0.015401599804560342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,65536,0.13074026902516683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,8192,0.028309333324432372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,10240,0.02959573268890381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,6144,0.02537173430124919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,12288,0.03409493366877238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,7168,0.026570665836334228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,16384,0.041892266273498534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,5120,0.02412160038948059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,4096,0.02223466634750366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,3584,0.020840533574422202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,3072,0.019997866948445638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,2560,0.018718934059143065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,2048,0.017849600315093993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,1536,0.018053332964579262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,768,0.01656426688035329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,1024,0.016807466745376587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,512,0.016341333587964378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,128,0.015475199619928996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,256,0.015625600020090738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,65536,0.11635413169860839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,32,0.015435733397801719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,64,0.015557333827018738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,7168,0.024728532632191977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,8192,0.026122667392094928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,12288,0.032545065879821776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,10240,0.029088000456492107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,6144,0.023397332429885863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,16384,0.03961706558863322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,5120,0.021868799130121866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,2560,0.018371200561523436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,4096,0.02081386645634969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,3584,0.019463467597961425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,3072,0.018453333775202432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,2048,0.017857066790262856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,1024,0.016691199938456216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,1536,0.017293866475423178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,65536,0.10517653624216716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,768,0.016184533635775246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,512,0.01597760021686554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,256,0.015470932920773825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,128,0.015388799707094827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,64,0.015388799707094827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,32,0.0153546671072642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,12288,0.03012053370475769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,16384,0.035037867228190106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,8192,0.024819199244181314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,10240,0.02723306616147359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,7168,0.023860265811284385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,6144,0.0227018674214681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,5120,0.021243733167648316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,4096,0.0194048007329305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,2048,0.0173909326394399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,2560,0.018273067474365235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,3584,0.01908479928970337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,3072,0.01780479947725932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,65536,0.0998517354329427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,768,0.016088533401489257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,1536,0.01698026657104492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,1024,0.016380799810091655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,512,0.01581546664237976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,256,0.015479466319084168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,64,0.015037866433461508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,128,0.015058133006095886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,32,0.015270400047302245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,12288,0.0298527995745341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,16384,0.03407359917958577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,6144,0.022030933698018392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,8192,0.024700800577799477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,10240,0.02704213261604309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,7168,0.02304640014966329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,3584,0.018863999843597413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,5120,0.02095253268877665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,4096,0.01915839910507202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,3072,0.018075732390085857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,65536,0.0856117328008016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,2560,0.01806079943974813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,2048,0.017052799463272095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,1536,0.016939733425776163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,1024,0.016145066420237223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,768,0.015819733341534935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,512,0.015558399756749473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,256,0.015364266435305276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,32,0.01514240006605784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,128,0.015149866541226705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,64,0.015082666277885437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,8192,0.02352213263511658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,12288,0.027124265829722088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,16384,0.030730666716893514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,10240,0.02441920042037964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,7168,0.0219925324122111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,6144,0.02068159977595011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,5120,0.0202239990234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,4096,0.019155200322469076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,3584,0.018914133310317993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,65536,0.07926399707794189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,3072,0.018529067436854042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,1536,0.016561067104339598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,2560,0.017859200636545815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,2048,0.017221333583196004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,1024,0.015943466623624166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,128,0.015086932977040609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,768,0.015974400440851848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,512,0.01569919983545939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,256,0.015416533748308817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,64,0.015253333250681558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,32,0.01502826710542043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,16384,0.029258666435877483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,10240,0.023797333240509033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,12288,0.025493333737055462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,8192,0.022513065735499063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,7168,0.022342399756113688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,6144,0.021396267414093017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,5120,0.020246400435765585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,65536,0.07416000366210937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,4096,0.019166932503382365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,3584,0.018760534127553304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,3072,0.018259199460347493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,2560,0.017809067169825235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,2048,0.017529600858688356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,1536,0.016935465733210246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,768,0.01581653356552124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,1024,0.016014933586120605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,512,0.01570453345775604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,256,0.0153546671072642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,32,0.015086932977040609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,64,0.015014400084813436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,128,0.015179733435312906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,16384,0.026183466116587322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,8192,0.02222826679547628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,10240,0.02360746661822001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,12288,0.024422399202982583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,7168,0.022222934166590373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,6144,0.02142826716105143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,65536,0.0698527971903483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,4096,0.019232000907262167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,5120,0.019883733987808228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,3584,0.018838399648666383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,3072,0.018270933628082277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,2560,0.017464532454808553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,1024,0.0160181333621343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,1536,0.016730666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,2048,0.017145599921544394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,768,0.015686399737993875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,512,0.015666133165359496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,256,0.015250133474667868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,64,0.015005866686503092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,128,0.01495039959748586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,32,0.0150709331035614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,16384,0.02614186604817708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,12288,0.024434133370717367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,8192,0.022261333465576173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,6144,0.021522132555643718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,10240,0.023550933599472045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,7168,0.022101332743962608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,65536,0.06545919974644979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,5120,0.01986453334490458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,4096,0.0192138671875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,3584,0.01863573392232259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,3072,0.01815999945004781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,2560,0.017474132776260375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,2048,0.017040000359217326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,1536,0.0167797327041626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,256,0.015280000368754067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,512,0.015564800302187601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,1024,0.015797332922617594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,768,0.015931733449300132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,128,0.015065600474675497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,64,0.014974932869275412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,32,0.015092266599337259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,16384,0.026340266068776447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,12288,0.02418773372968038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,10240,0.023803732792536416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,5120,0.019901865720748903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,8192,0.022177066405614218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,65536,0.06268800099690755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,6144,0.021308799584706627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,7168,0.02196906606356303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,4096,0.018768000602722167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,3584,0.01889386574427287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,3072,0.01809813380241394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,2048,0.017069866259892784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,2560,0.01761066714922587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,1536,0.016770132382710776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,1024,0.015972266594568886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,768,0.015687466661135355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,512,0.015705600380897522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,256,0.01513920029004415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,128,0.015061333775520325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,64,0.015080533425013223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,32,0.015072000026702882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,5120,0.1427445411682129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,6144,0.1664405345916748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,7168,0.1896618684132894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,8192,0.2152714729309082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,10240,0.2606880029042562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,2560,0.08441600004831949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,4096,0.1199839989344279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,3584,0.10808746814727783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,3072,0.0964085340499878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,12288,0.3077397346496582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,512,0.03439146677652995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,768,0.04159146547317505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,256,0.027378133932749432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,1024,0.04778560002644856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,1536,0.05980480114618937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,2048,0.07210240364074708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,128,0.023614933093388878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,64,0.02323413292566935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,32,0.022858667373657226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,16384,0.4029685338338216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,10240,0.08063573042551676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,7168,0.061761065324147546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,8192,0.06801599661509196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,12288,0.09301119645436605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,16384,0.1166922648747762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,6144,0.0619050661722819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,3584,0.040411734580993654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,5120,0.0499232014020284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,4096,0.04358933369318645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,3072,0.03739200035730998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,2560,0.03370986779530843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,1024,0.022590933243433635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,768,0.02107093334197998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,1536,0.02601813276608785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,2048,0.029890133937199907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,512,0.019729065895080566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,128,0.016942934195200602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,256,0.01741546591122945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,64,0.017359999815622966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,32,0.017786665757497152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,7168,0.05194133520126343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,10240,0.06684373219807943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,8192,0.05637333393096924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,12288,0.07712639967600504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,16384,0.09511893590291341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,4096,0.03680213292439778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,5120,0.041980799039204916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,6144,0.046728531519571945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,3584,0.03454826672871907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,1024,0.021011199553807577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,3072,0.03177066644032796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,2048,0.02574186722437541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,2560,0.028642133871714277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,1536,0.02322559952735901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,768,0.019921066363652547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,256,0.01679253379503886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,64,0.016703999042510985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,128,0.016250666975975037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,512,0.017544533809026083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,32,0.01704533298810323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,8192,0.051076265176137295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,10240,0.061161601543426515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,65536,0.40352748235066727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,12288,0.0695637305577596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,16384,0.08519252936045328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,7168,0.0469866673151652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,6144,0.04266560077667236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,4096,0.03410773277282715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,5120,0.04177600145339966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,3584,0.0320032000541687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,3072,0.02912000020345052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,2560,0.026778666178385417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,2048,0.024046933650970458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,768,0.019629865884780884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,1024,0.020426666736602782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,1536,0.022655999660491942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,65536,0.34310614267985023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,128,0.016420267025629678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,512,0.017832533518473307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,256,0.01684266726175944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,64,0.01660266617933909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,32,0.01672640045483907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,8192,0.04425813357035319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,10240,0.05127466519673666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,7168,0.04071679910024007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,16384,0.0720362663269043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,12288,0.058627200126647946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,3584,0.027739733457565308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,3072,0.02579093376795451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,4096,0.029735465844472248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,6144,0.03749866485595703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,5120,0.033879466851552326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,65536,0.30622933705647787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,2560,0.023861332734425863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,2048,0.022270933787027995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,256,0.016364799936612447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,1024,0.019079466660817467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,768,0.01760639945665995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,1536,0.020883200565973918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,512,0.016195199886957803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,128,0.01602453291416168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,64,0.016315733393033348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,32,0.01672853430112203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,8192,0.040956799189249674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,12288,0.0542143980662028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,7168,0.03841813405354817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,10240,0.04792533318201701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,16384,0.06590826511383056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,4096,0.027677865823109944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,3584,0.02630079984664917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,5120,0.03150933384895325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,6144,0.03519466718037923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,65536,0.2423093318939209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,3072,0.024717867374420166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,2560,0.023149865865707397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,1536,0.020244266589482626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,1024,0.018433066209157307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,2048,0.0217738668123881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,768,0.017012266318003337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,512,0.016721065839131674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,64,0.01619733373324076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,128,0.0161461333433787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,256,0.016169599692026772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,32,0.016377600034077962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,8192,0.03835306564966838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,10240,0.04488213459650676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,7168,0.036023465792338054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,12288,0.05070933500925699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,16384,0.06092906792958578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,65536,0.2283189296722412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,5120,0.02985600034395854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,4096,0.0264575997988383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,3584,0.025111466646194458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,6144,0.03282879988352458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,3072,0.02378986676534017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,1024,0.01728106737136841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,2560,0.02258453369140625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,2048,0.021350399653116862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,1536,0.019891200462977092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,768,0.017359999815622966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,256,0.016214399536450704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,512,0.01673706571261088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,64,0.01607039968172709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,128,0.015705600380897522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,32,0.016238933801651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,10240,0.04176106850306193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,7168,0.03366080125172933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,8192,0.035892268021901444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,12288,0.046834135055541994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,16384,0.056278399626413976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,65536,0.20432213147481285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,6144,0.030483200152715044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,4096,0.024922666947046916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,5120,0.028104533751805622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,3072,0.0228767991065979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,3584,0.024327466885248818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,2560,0.022098133961359658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,2048,0.02138239940007528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,768,0.01726400057474772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,1024,0.017308799425760905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,1536,0.019386667013168334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,512,0.01694399913152059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,256,0.01602666676044464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,64,0.01581760048866272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,128,0.015842133760452272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,32,0.01590079963207245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,7168,0.028837333122889202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,8192,0.03127040068308513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,10240,0.03750613530476888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,65536,0.1754591941833496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,12288,0.04862186511357625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,16384,0.05575573444366455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,5120,0.024791467189788818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,6144,0.02730453411738078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,4096,0.023125332593917847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,3584,0.022290132443110146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,3072,0.021492266654968263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,2560,0.020488532384236653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,2048,0.019357866048812865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,1024,0.01717546582221985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,1536,0.017799466848373413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,768,0.016902399063110352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,128,0.015522133310635886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,512,0.016452266772588094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,256,0.016042666633923848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,64,0.01588159998257955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,32,0.016078933080037435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,65536,0.15221333503723145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,8192,0.029410133759180706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,7168,0.027454932530721027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,10240,0.03428479830423991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,12288,0.038541865348815915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,5120,0.023706666628519692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,6144,0.025123200813929242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,16384,0.046402132511138915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,4096,0.022129066785176597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,2560,0.019833600521087645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,2048,0.01832533280054728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,3584,0.021492266654968263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,1536,0.016772266228993735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,3072,0.0206496000289917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,1024,0.016474666198094685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,768,0.01654293338457743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,128,0.015731199582417806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,512,0.016076800227165223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,256,0.015718400478363037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,64,0.01569066643714905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,32,0.015830399592717488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,65536,0.13992533683776856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,16384,0.04934826691945394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,10240,0.03664213418960571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,8192,0.030501333872477214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,7168,0.026445867617925008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,6144,0.02482879956563314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,12288,0.040491731961568196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,5120,0.02336853345235189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,3072,0.020465066035588585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,4096,0.0217738668123881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,3584,0.0209279994169871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,2048,0.01772586703300476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,2560,0.019512534141540527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,1536,0.017285333077112833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,64,0.015583999951680503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,1024,0.016446933150291443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,128,0.01572480003039042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,256,0.015430399775505066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,768,0.016370133558909098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,512,0.01588373382886251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,65536,0.1304362694422404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,32,0.01567466656366984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,12288,0.037638401985168456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,8192,0.029256532589594524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,6144,0.02582293351491292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,10240,0.033854933579762776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,7168,0.027880533536275225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,16384,0.04562453428904216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,3584,0.02068906625111898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,5120,0.022402133544286093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,2560,0.018563199043273925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,4096,0.021134932835896812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,3072,0.019963733355204263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,2048,0.017382399241129557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,768,0.01641493340333303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,1536,0.017205333709716795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,1024,0.01637440025806427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,512,0.015997866789499916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,256,0.015685333808263143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,128,0.015449600418408713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,64,0.015500799814860026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,65536,0.11646613279978435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,32,0.015364266435305276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,10240,0.032409600416819256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,8192,0.028458666801452637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,12288,0.03637866576512654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,6144,0.025202133258183795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,7168,0.026844799518585205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,16384,0.04321386814117432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,3072,0.019502933820088705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,5120,0.022272000710169472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,4096,0.020916267236073812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,3584,0.02034239967664083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,2560,0.018145066499710084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,2048,0.01773866613705953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,1536,0.01693120002746582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,768,0.01620693306128184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,1024,0.016401066382726034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,65536,0.10475733280181884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,256,0.015691733360290526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,512,0.01599360009034475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,128,0.015176533659299215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,64,0.015498666961987814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,32,0.015706666310628257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,16384,0.041317331790924075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,12288,0.03392533461252849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,10240,0.030525867144266767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,7168,0.025763199726740522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,6144,0.024193066358566283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,8192,0.027266132831573486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,5120,0.02114880084991455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,4096,0.019450666507085164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,2560,0.017911465962727864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,3072,0.01874879995981852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,3584,0.01952106753985087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,2048,0.017442133029301962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,65536,0.09354453086853028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,768,0.016133333245913185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,512,0.01581546664237976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,1536,0.016820265849431356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,1024,0.01634986698627472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,256,0.015498666961987814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,128,0.015166933337847391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,32,0.015440000096956888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,64,0.015352533260981242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,12288,0.028908799091974895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,16384,0.033879466851552326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,7168,0.02332906723022461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,10240,0.026953599850336712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,8192,0.024327466885248818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,6144,0.02215999960899353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,3584,0.019523199399312338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,5120,0.02098133365313212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,4096,0.019568000237147012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,2048,0.017388800779978432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,3072,0.01867199937502543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,65536,0.08107840220133464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,1536,0.016927999258041383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,2560,0.018246400356292724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,1024,0.01636373301347097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,768,0.01600106656551361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,64,0.015278933445612588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,512,0.01583999991416931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,128,0.015186132987340293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,256,0.015477333466211954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,32,0.015090133746465048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,16384,0.03212053378423055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,12288,0.02877440055211385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,8192,0.024254933993021647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,10240,0.026184533039728803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,7168,0.02241386572519938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,6144,0.020994132757186888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,4096,0.01923840045928955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,5120,0.01990399956703186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,3584,0.01867626706759135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,3072,0.018296533823013307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,65536,0.07587093512217204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,2560,0.017926400899887084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,2048,0.017169066270192466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,768,0.01602026621500651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,1536,0.0169322669506073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,1024,0.015983999768892924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,512,0.015802666544914246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,256,0.015396266182263692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,32,0.01516480048497518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,128,0.015230933825174967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,64,0.015126400192578635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,16384,0.03027733365694682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,8192,0.024196267127990723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,12288,0.027020800113677978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,10240,0.025256532430648803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,7168,0.022753065824508666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,6144,0.02113706668217977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,4096,0.01892906626065572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,65536,0.07457813421885172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,5120,0.019831466674804687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,3584,0.01845226685206095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,3072,0.01813439925511678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,1536,0.01656426688035329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,2560,0.01765120029449463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,2048,0.01704533298810323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,512,0.015637333194414772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,768,0.015829333662986757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,1024,0.016089600324630738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,128,0.015102932850519816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,256,0.01541973352432251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,64,0.015191466609636942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,32,0.014997333288192749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,10240,0.023217066129048666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,16384,0.026498132944107057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,12288,0.023995733261108397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,8192,0.02203413248062134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,5120,0.019698133071263633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,65536,0.06582186619440714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,7168,0.021918932596842446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,6144,0.021146667003631592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,4096,0.019032533963521323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,3584,0.018541866540908815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,3072,0.01811520059903463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,2560,0.017383466164271034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,2048,0.01695466637611389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,1536,0.016500266393025716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,1024,0.01590720017751058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,128,0.015085867047309876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,768,0.015920000274976094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,512,0.015611732999483744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,256,0.015406933426856995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,64,0.015053866306940713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,32,0.01511146624883016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,16384,0.02537386616071065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,10240,0.022906666994094847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,12288,0.023105067014694215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,8192,0.021771732966105142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,7168,0.02188053329785665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,5120,0.01981333295504252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,6144,0.02102186679840088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,65536,0.05834559996922811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,4096,0.018873600165049235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,3584,0.018122667074203493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,3072,0.01804693341255188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,2560,0.017348267634709678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,2048,0.017058134078979492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,768,0.016035200158754984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,1536,0.016404267152150473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,1024,0.01589120030403137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,512,0.015397333105405173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,256,0.015375999609629312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,64,0.015069866180419922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,128,0.015160533785820007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,32,0.015106133619944253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,12288,0.023041067520777385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,8192,0.021637332439422608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,16384,0.025282132625579833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,10240,0.02304853399594625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,7168,0.021733333667119346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,6144,0.02111146648724874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,65536,0.053855999310811364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,5120,0.019826134045918785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,2048,0.01691840092341105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,4096,0.01873706579208374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,2560,0.017387733856836955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,3072,0.01790613333384196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,3584,0.01821546753247579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,1536,0.01653333306312561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,768,0.015889066457748412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,512,0.015362133582433065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,1024,0.01581653356552124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,256,0.01525973379611969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,128,0.015190399686495461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,64,0.01497706671555837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,32,0.015069866180419922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,16384,0.02534293333689372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,12288,0.023040000597635904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,8192,0.02183039983113607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,10240,0.02290346622467041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,7168,0.0216757337252299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,5120,0.019817600647608437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,65536,0.05133973360061646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,6144,0.02104319930076599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,4096,0.01857173244158427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,3584,0.018563199043273925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,3072,0.018027732769648232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,2048,0.016808533668518068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,2560,0.017571200927098594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,1536,0.016839466492335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,1024,0.016109866897265117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,128,0.015040000279744467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,768,0.015635200341542563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,512,0.015576533476511636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,256,0.015196800231933594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,32,0.015030399958292643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,64,0.014944000045458474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,5120,0.139682133992513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,6144,0.16262507438659668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,7168,0.18509653409322102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,8192,0.2100202719370524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,10240,0.2545802593231201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,3584,0.10480213165283203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,3072,0.09351360003153483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,4096,0.1170357306798299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,12288,0.3014464060465495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,2560,0.0816490650177002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,1536,0.057633066177368165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,512,0.03362773259480794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,256,0.025179733832677204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,768,0.040040532747904464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,2048,0.06955093542734782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,1024,0.04554239908854167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,128,0.022754132747650146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,64,0.021615999937057494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,32,0.021689599752426146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,16384,0.3939776102701823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,8192,0.06666666666666667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,7168,0.06066346565882365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,10240,0.07813653151194254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,12288,0.09041600227355957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,16384,0.11370133558909098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,6144,0.060227199395497644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,3584,0.03939733505249023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,4096,0.04230613311131795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,5120,0.04876373211542766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,3072,0.036177066961924235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,1536,0.025166932741800947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,768,0.020844799280166627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,1024,0.02205866575241089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,512,0.019324799378712974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,2560,0.03294080098470052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,2048,0.02905279994010925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,256,0.01690666675567627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,128,0.016684800386428833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,64,0.017138133446375527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,32,0.017222400506337485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,7168,0.050134400526682533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,10240,0.06405653158823649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,8192,0.0547104001045227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,12288,0.07342507044474283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,16384,0.09062186876932779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,3584,0.0333898663520813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,4096,0.035835735003153485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,5120,0.04059733152389526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,6144,0.04557013511657715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,2560,0.027893332640329997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,3072,0.031163734197616578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,1536,0.022499199708302817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,2048,0.025035732984542848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,1024,0.020434133211771646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,768,0.01938026746114095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,256,0.016478932897249856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,512,0.01726079980532328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,128,0.016239999731381734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,64,0.016733866930007935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,32,0.01665279964605967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,8192,0.04883306821187337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,10240,0.05686506827672323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,12288,0.06547093391418457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,16384,0.07991999785105387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,65536,0.3988149325052897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,7168,0.04463253418604533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,6144,0.040906667709350586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,5120,0.03694399992624919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,3072,0.027798400322596235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,4096,0.03255786697069804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,3584,0.03297173380851746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,1536,0.021781333287556968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,2560,0.025684267282485962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,2048,0.023612799247105916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,1024,0.01965973377227783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,65536,0.32632853190104166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,512,0.01708266735076904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,768,0.018634666999181114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,256,0.016391467054684958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,128,0.016127999623616537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,64,0.01653866668542226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,32,0.015893333156903586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,7168,0.03906559944152832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,12288,0.05675093332926432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,8192,0.043399465084075925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,10240,0.04949013392130534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,16384,0.06889493465423584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,65536,0.2924714724222819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,3584,0.02693333427111308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,5120,0.03255680004755656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,6144,0.0360586682955424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,3072,0.02527359922726949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,4096,0.029021867116292316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,2560,0.024973867336908977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,2048,0.021926399072011313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,256,0.01601599951585134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,1536,0.02035413384437561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,1024,0.01867093245188395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,512,0.016883200407028197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,768,0.01707306702931722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,64,0.016134400169054666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,128,0.01573973298072815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,32,0.016152532895406087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,10240,0.04640640020370483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,8192,0.039988267421722415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,7168,0.036321067810058595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,12288,0.05277653137842814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,16384,0.06312533219655356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,65536,0.23342612584431968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,4096,0.027012266715367633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,6144,0.03369919856389363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,3584,0.02537493308385213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,5120,0.03076266646385193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,3072,0.023844265937805177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,2560,0.02320853273073832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,2048,0.021222400665283202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,1536,0.01975040038426717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,768,0.01686826745669047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,1024,0.017612799008687337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,512,0.016522666811943053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,256,0.015898666779200234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,64,0.015735466281572977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,128,0.015762133399645488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,32,0.01590826710065206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,12288,0.04785386721293132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,10240,0.04198613166809082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,8192,0.036925868193308516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,7168,0.0336949348449707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,16384,0.05741333166758219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,65536,0.20952107111612955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,5120,0.028514132897059126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,4096,0.025256532430648803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,3584,0.023832533756891885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,6144,0.031108266115188597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,3072,0.022780799865722658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,2048,0.020482132832209267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,2560,0.02142826716105143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,1536,0.019335466623306274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,1024,0.01662399967511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,256,0.015692800283432007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,768,0.016522666811943053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,512,0.016425599654515587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,128,0.015373866756757101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,64,0.01590933303038279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,32,0.015994667013486227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,16384,0.05229546626408895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,10240,0.03796906471252441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,12288,0.043673598766326906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,8192,0.033478399117787674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,7168,0.03111039996147156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,65536,0.20098773638407388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,6144,0.0283786674340566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,5120,0.025361067056655882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,3584,0.02281600038210551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,3072,0.02181546688079834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,4096,0.023676800727844238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,2560,0.02076266606648763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,2048,0.019718400637308755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,128,0.015518933534622192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,1536,0.01736746629079183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,1024,0.016471466422080992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,768,0.01653866668542226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,512,0.01627306640148163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,256,0.01574399967988332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,64,0.015546666582425437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,32,0.01535146633783976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,65536,0.16784000396728516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,7168,0.027727999289830524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,8192,0.030326400200525922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,12288,0.038940799236297605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,10240,0.03443520069122315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,16384,0.04688959916432699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,6144,0.0256661335627238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,4096,0.022553600867589316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,5120,0.023831466833750405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,3584,0.021617066860198975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,2560,0.020114133755366005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,3072,0.020808533827463786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,1536,0.01704960068066915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,2048,0.019139200448989868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,1024,0.01593706707159678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,512,0.016028799613316855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,768,0.016501333316167197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,128,0.015401599804560342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,256,0.01554026703039805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,64,0.015758933623631795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,32,0.015686399737993875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,65536,0.1429482618967692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,8192,0.028233599662780762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,7168,0.026270933945973712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,12288,0.036823467413584395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,10240,0.03262826601664225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,5120,0.0236735999584198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,6144,0.02472426692644755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,16384,0.043944533665974936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,4096,0.022000000874201456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,2560,0.01986666719118754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,3584,0.02119999925295512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,3072,0.0206986665725708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,2048,0.018259199460347493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,1536,0.01687999963760376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,768,0.01623466710249583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,1024,0.01641706625620524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,512,0.01611306667327881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,128,0.015442132949829102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,32,0.015524267156918844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,256,0.01567359964052836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,64,0.015587199727694193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,65536,0.13194879690806072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,10240,0.030598400036493938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,8192,0.026807467142740887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,7168,0.025352533658345538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,12288,0.03454933166503906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,16384,0.04086933135986328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,5120,0.02287999987602234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,6144,0.024013866980870567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,2048,0.017787732680638633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,3584,0.021205333868662517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,4096,0.021284266312917074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,3072,0.020103466510772706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,2560,0.019008000691731773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,1536,0.016999467213948568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,1024,0.016526933511098227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,128,0.01546346644560496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,256,0.015948800245920818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,768,0.016396799683570863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,512,0.016251732905705772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,64,0.015611732999483744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,32,0.015267200271288552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,65536,0.11871253649393718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,7168,0.02438186605771383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,10240,0.02841493288675944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,12288,0.03210879961649577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,6144,0.022973867257436116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,8192,0.02592960000038147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,16384,0.039191468556722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,5120,0.021924267212549843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,4096,0.02085439960161845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,3584,0.02028800050417582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,1536,0.016804265975952148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,3072,0.019237333536148073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,2560,0.018450133005777993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,2048,0.017170133193333943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,512,0.01606933375199636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,1024,0.016407466928164163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,256,0.01537493367989858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,768,0.016389333208402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,128,0.015332266688346863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,64,0.015427199999491372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,32,0.015361066659291586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,65536,0.1082154671351115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,8192,0.025614933172861738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,10240,0.029704533020655316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,7168,0.024396799008051553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,12288,0.0315285325050354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,6144,0.023257599274317423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,16384,0.03813440004984538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,3584,0.01885120073954264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,5120,0.021348265806833903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,4096,0.020172800620396933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,2560,0.017483733097712197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,3072,0.018438400824864705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,2048,0.01712426741917928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,1536,0.016821332772572837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,512,0.015838932991027833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,1024,0.016247466206550598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,768,0.01609493295351664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,256,0.015505066514015198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,128,0.0150218665599823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,65536,0.09748906294504801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,64,0.015244799852371215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,32,0.015261866648991904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,16384,0.03363093137741089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,10240,0.025835732618967693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,12288,0.028034132719039918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,6144,0.021784534056981407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,8192,0.023695999383926393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,7168,0.022574933369954427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,5120,0.020837332804997763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,4096,0.019668267170588175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,3072,0.01831573247909546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,2048,0.017080533504486083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,3584,0.01843199928601583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,2560,0.017913599809010826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,65536,0.08401280244191488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,1536,0.016819200913111367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,768,0.016035200158754984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,512,0.01560533344745636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,1024,0.01602026621500651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,256,0.015520000457763672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,128,0.015178666512171427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,64,0.015176533659299215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,32,0.015310933192571005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,16384,0.029284266630808513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,12288,0.026499199867248534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,5120,0.020091732343037925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,7168,0.023088000218073525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,10240,0.02553173303604126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,8192,0.023897600173950196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,6144,0.02122453252474467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,4096,0.018708266814549766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,3584,0.018294399976730345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,65536,0.07354986667633057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,3072,0.018155733744303383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,2560,0.017545600732167564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,2048,0.016960000991821288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,1536,0.01662506659825643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,768,0.01585599978764852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,1024,0.01595093309879303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,512,0.01572160025437673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,256,0.015681067109107973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,128,0.015221333503723145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,32,0.015245866775512696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,64,0.014995200435320535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,12288,0.025319466988245647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,16384,0.028282666206359865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,10240,0.02412373423576355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,7168,0.021347200870513915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,8192,0.02286293307940165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,6144,0.0201749324798584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,5120,0.019796266158421835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,65536,0.06657386620839437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,4096,0.01858133276303609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,3584,0.018523732821146645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,1536,0.01660053332646688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,3072,0.017831466595331826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,2560,0.017568000157674155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,2048,0.016908800601959227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,1024,0.0160480002562205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,768,0.01572266618410746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,256,0.015307733416557312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,512,0.015686399737993875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,128,0.015134933590888976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,64,0.01516266663869222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,32,0.01499626636505127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,12288,0.025218133131663007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,16384,0.026292266448338826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,10240,0.023067732652028404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,8192,0.021821866432825722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,5120,0.019694934288660683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,6144,0.02062186598777771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,7168,0.021025067567825316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,65536,0.0603488008181254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,4096,0.018786134322484334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,3584,0.018576000134150186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,2560,0.017372800906499227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,3072,0.017846399545669557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,1024,0.01609813372294108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,2048,0.01687039931615194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,768,0.01571626663208008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,1536,0.016544000307718913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,256,0.015371732910474143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,512,0.015717333555221556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,128,0.015233066678047181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,64,0.015401599804560342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,32,0.015331199765205384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,12288,0.02235306700070699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,10240,0.022232532501220703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,16384,0.02422719995180766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,8192,0.021077332894007365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,65536,0.054819198449452725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,5120,0.019687465826670327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,7168,0.021176532904307047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,6144,0.020581332842508952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,4096,0.018577067057291667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,3072,0.017832533518473307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,3584,0.018428800503412883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,2048,0.017093332608540852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,2560,0.01725546717643738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,1536,0.016475733121236166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,768,0.015818666418393454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,1024,0.015961600343386333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,128,0.015124266346295675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,512,0.015662933389345803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,256,0.015383467078208923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,64,0.014910933375358582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,32,0.01498240033785502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,12288,0.022498132785161336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,10240,0.02248959938685099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,16384,0.02416320045789083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,7168,0.021232000986735024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,6144,0.020786132415135702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,8192,0.021090133984883627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,65536,0.04674026568730672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,5120,0.019610667228698732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,4096,0.018760534127553304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,3072,0.01764799952507019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,2560,0.01730239987373352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,1024,0.015965867042541503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,3584,0.018262400229771932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,1536,0.016328533490498863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,2048,0.01712533235549927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,768,0.015844266613324484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,128,0.015105066696802774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,512,0.015410133202870688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,256,0.01527679959932963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,32,0.01495039959748586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,64,0.014879999558130899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,12288,0.022163200378417968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,16384,0.024075732628504435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,8192,0.020770132541656494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,10240,0.02169813315073649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,65536,0.043687466780344644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,7168,0.021223467588424683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,6144,0.020459733406702676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,5120,0.019479467471440633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,3584,0.018206934134165444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,4096,0.018680532773335777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,3072,0.017653334140777587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,2048,0.017155200242996216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,2560,0.017668267091115318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,1536,0.016403200229008992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,768,0.0157258669535319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,1024,0.015913599729537965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,512,0.015683199961980185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,128,0.015064533551534018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,256,0.015031466881434122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,64,0.015090133746465048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,16384,0.023715200026830037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,32,0.014851199587186179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,12288,0.02224853237469991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,10240,0.021898667017618813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,8192,0.02069759964942932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,6144,0.020292266209920248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,65536,0.04299519856770833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,7168,0.02110613385836283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,5120,0.019434666633605956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,2560,0.01729066570599874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,4096,0.018659200270970663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,3584,0.018308266003926595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,3072,0.017742933829625447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,2048,0.01695466637611389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,1536,0.016376533110936484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,768,0.015734400351842245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,1024,0.015982932845751443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,512,0.01546346644560496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,256,0.015152000387509666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,64,0.014919466773668923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,128,0.014781866470972696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,32,0.014974932869275412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,5120,0.1353962739308675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,6144,0.15792214075724284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,7168,0.18066666920979818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,8192,0.20499520301818847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,3072,0.09007786909739177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,10240,0.24854079882303873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,4096,0.11283093293507893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,3584,0.10075093110402425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,2560,0.07869119644165039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,12288,0.29700374603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,768,0.03756693204243978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,1024,0.04363199869791667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,2048,0.06669226487477621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,1536,0.05511786540349325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,512,0.030665600299835206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,256,0.024171733856201173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,128,0.02111146648724874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,32,0.019859200716018675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,64,0.01983893314997355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,16384,0.38335361480712893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,8192,0.06450239817301431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,7168,0.0586517333984375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,10240,0.0755295991897583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,12288,0.08793173631032308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,6144,0.05832746823628744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,16384,0.11002453168233235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,5120,0.04715733528137207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,4096,0.04109866619110107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,3584,0.03792426586151123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,2560,0.03206933339436849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,3072,0.03509866793950399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,1536,0.024892799059549966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,2048,0.0283135990301768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,1024,0.021898667017618813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,128,0.016219733158747356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,768,0.020163200298945107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,512,0.01893226703008016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,256,0.016582399606704712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,64,0.01655893325805664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,32,0.01676266590754191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,8192,0.05394986470540365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,7168,0.04906986554463704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,12288,0.07082346280415854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,10240,0.06288640101750692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,16384,0.08928319613138834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,4096,0.03550293445587158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,5120,0.04036800066630046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,6144,0.045125333468119304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,3584,0.03325653274854024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,2560,0.0274399995803833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,3072,0.030746666590372722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,1536,0.02285226583480835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,768,0.019387733936309815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,2048,0.025272534290949507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,1024,0.020814933379491172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,512,0.0168938676516215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,128,0.015825066963831583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,256,0.016361600160598753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,64,0.016343466440836587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,32,0.01621119976043701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,8192,0.04750293493270874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,12288,0.06330666542053223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,16384,0.07701866626739502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,65536,0.3825023969014486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,10240,0.05576106707255045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,5120,0.03630186716715495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,4096,0.03448959986368815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,3584,0.03006826639175415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,6144,0.04015466769536336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,7168,0.04374186595280965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,3072,0.027827199300130206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,65536,0.31382614771525064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,2560,0.025438932577768962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,2048,0.023461333910624185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,1536,0.02172799905141195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,768,0.01853013237317403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,1024,0.019819732507069907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,32,0.01598186691602071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,256,0.016375466187795003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,512,0.016663466890652977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,64,0.016057599584261575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,128,0.01602453291416168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,10240,0.04754453500111898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,8192,0.04139413436253865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,7168,0.03818986813227336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,12288,0.054406400521596274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,16384,0.06615466674168904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,3584,0.02650773326555888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,4096,0.028344533840815228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,65536,0.27916266123453776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,5120,0.032034132877985635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,6144,0.03510186672210693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,3072,0.024846933285395303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,512,0.016645333170890807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,2560,0.0232149342695872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,1024,0.018458666404088338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,768,0.01693013310432434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,1536,0.02087786595026652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,2048,0.021601066986719767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,256,0.01597760021686554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,64,0.016151466965675355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,128,0.015846400459607442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,32,0.01609813372294108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,7168,0.035646935304005936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,8192,0.038678399721781415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,12288,0.05013120174407959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,10240,0.044691201051076254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,16384,0.06057386795679728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,4096,0.026369067033131917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,65536,0.22339839935302735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,3584,0.02507413427035014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,6144,0.033004800478617355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,3072,0.023721599578857423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,5120,0.029897600412368774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,2560,0.022422399123509726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,2048,0.021851734320322672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,1536,0.019614932934443156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,1024,0.017034665743509928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,256,0.01577173372109731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,768,0.016173866391181946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,512,0.016310399770736693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,32,0.01569386621316274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,64,0.01579839984575907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,128,0.015761066476504007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,10240,0.040491731961568196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,12288,0.04631893237431844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,8192,0.035640533765157065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,7168,0.033004800478617355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,16384,0.05501653353373209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,65536,0.2095359961191813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,6144,0.030628265937169392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,5120,0.027592533826828004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,4096,0.02492693265279134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,3584,0.023808000485102336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,3072,0.022536534070968627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,2560,0.02170133392016093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,2048,0.020165334145228066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,1024,0.016380799810091655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,1536,0.019272534052530925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,128,0.01562346617380778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,256,0.015614933768908181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,768,0.016334933042526246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,512,0.016229333480199178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,64,0.01580586632092794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,32,0.015843199690183003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,10240,0.03751786549886067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,65536,0.18177812894185383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,12288,0.041699198881785075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,8192,0.03302293419837952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,7168,0.030542933940887453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,16384,0.05013120174407959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,5120,0.025397332509358723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,6144,0.027688533067703247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,3584,0.022407466173172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,3072,0.021566933393478392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,2560,0.020606933037439983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,4096,0.0233024001121521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,2048,0.019497599204381308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,128,0.015485866864522298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,1024,0.01662826637427012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,1536,0.017968000968297322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,512,0.01618026693662008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,768,0.016214399536450704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,256,0.01557866632938385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,64,0.015588266650835672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,32,0.015546666582425437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,65536,0.15720853805541993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,7168,0.0271232008934021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,10240,0.033515731493632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,12288,0.04192320108413696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,8192,0.029203200340270997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,16384,0.04937386512756348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,6144,0.02518186569213867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,3072,0.02062293291091919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,5120,0.023705599705378215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,3584,0.021385600169499717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,2560,0.019844265778859456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,4096,0.02242986758550008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,2048,0.018583466609319053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,1536,0.016679465770721436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,1024,0.016424533724784852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,768,0.016320000092188515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,512,0.015752533078193666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,256,0.015440000096956888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,128,0.015277866522471109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,64,0.01569493313630422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,32,0.01576746702194214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,65536,0.1351274649302165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,8192,0.027643734216690065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,7168,0.025415466229120894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,10240,0.03118079900741577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,16384,0.04604906638463338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,12288,0.036177066961924235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,5120,0.02295893430709839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,6144,0.024502400557200113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,4096,0.021253333489100138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,3584,0.021005866924921672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,2048,0.01808639963467916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,3072,0.020051199197769164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,2560,0.019413334131240845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,1536,0.016378666957219443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,512,0.015988266468048094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,1024,0.016270933548609416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,768,0.016088533401489257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,256,0.015429332852363586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,128,0.015261866648991904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,64,0.015548800428708395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,32,0.01546346644560496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,65536,0.12405973275502521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,7168,0.024615466594696045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,8192,0.02592960000038147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,10240,0.02914773424466451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,12288,0.03293866713841756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,16384,0.042954667409261064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,6144,0.023447465896606446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,5120,0.022275199492772423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,4096,0.020898133516311646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,2560,0.017909334103266398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,3072,0.019525333245595296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,3584,0.02035520076751709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,2048,0.01714026729265849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,1536,0.016822399695714314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,64,0.015399466951688132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,1024,0.016521599888801575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,768,0.01628266672293345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,512,0.015811199943224587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,128,0.015404799580574035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,256,0.015470932920773825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,65536,0.11322773297627767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,32,0.015402666727701821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,12288,0.032150399684906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,10240,0.029101866483688354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,7168,0.024741333723068238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,8192,0.025938133398691814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,6144,0.023881600300470988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,16384,0.03923733234405517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,2560,0.018230400482813516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,3584,0.02016426722208659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,4096,0.02074880003929138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,5120,0.0213536004225413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,3072,0.01839253306388855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,2048,0.017257599035898845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,1536,0.016561067104339598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,1024,0.01618026693662008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,512,0.015759999553362526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,768,0.01618133286635081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,128,0.015236266454060874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,256,0.015452800194422403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,64,0.01530880033969879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,65536,0.10256106853485107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,32,0.015340800086657206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,12288,0.028642133871714277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,8192,0.024065067370732628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,10240,0.026206932465235394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,16384,0.03547946612040202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,7168,0.02322346568107605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,6144,0.022156800826390585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,4096,0.019973333676656088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,5120,0.020984532435735066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,3584,0.019528534015019736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,2048,0.017177599668502807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,3072,0.017896533012390137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,2560,0.017026132345199584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,1536,0.016613333423932394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,1024,0.016201600432395935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,65536,0.08698346614837646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,512,0.015763200322786965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,768,0.015871999661127727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,256,0.015430399775505066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,128,0.015313067038853965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,64,0.015431466698646545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,32,0.015301332871119181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,6144,0.02143893241882324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,16384,0.031202133496602374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,7168,0.02255893349647522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,10240,0.025130667289098102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,8192,0.023229867219924927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,12288,0.026824533939361572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,5120,0.020521599054336547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,3584,0.018169599771499633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,4096,0.01885120073954264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,3072,0.017947733402252197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,2560,0.017496534188588462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,2048,0.01699519952138265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,65536,0.07683093547821045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,1536,0.01667413314183553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,1024,0.016156799594561257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,768,0.015960533420244852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,512,0.015778133273124696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,256,0.015506133437156677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,128,0.015082666277885437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,32,0.01518186628818512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,64,0.015085867047309876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,16384,0.028696533044179278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,7168,0.02207039992014567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,12288,0.02609279950459798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,10240,0.024609067042668662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,6144,0.0210698664188385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,8192,0.023439999421437582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,4096,0.018399999539057414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,5120,0.0196288009484609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,65536,0.0647872010866801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,3584,0.018489599227905273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,2560,0.017121066649754844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,3072,0.01791999936103821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,1536,0.016407466928164163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,2048,0.01684479912122091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,1024,0.016007467110951742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,768,0.01586133340994517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,512,0.015756799777348836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,64,0.015158399939537048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,256,0.015175466736157736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,128,0.015209600329399109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,32,0.015100799997647605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,10240,0.023110399643580117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,12288,0.025286400318145753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,16384,0.02775999903678894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,8192,0.021717333793640138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,7168,0.020857600371042888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,6144,0.020693333943684895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,5120,0.01971413294474284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,65536,0.05922026634216308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,4096,0.01886826753616333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,3584,0.018172800540924072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,3072,0.017670400937398276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,1536,0.016354133685429893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,2560,0.017326933145523072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,2048,0.016784000396728515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,768,0.0159850666920344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,1024,0.01618026693662008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,512,0.015682133038838704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,128,0.014959999918937683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,256,0.015108266472816467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,64,0.01515733301639557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,32,0.015043200055758158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,10240,0.02299413283665975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,8192,0.020804266134897866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,12288,0.023067732652028404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,16384,0.025729066133499144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,5120,0.019361066818237304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,7168,0.020814933379491172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,6144,0.020363734165827433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,65536,0.053659733136494955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,4096,0.018572799364725747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,3584,0.01807360053062439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,3072,0.01784106691678365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,2560,0.017248000701268515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,2048,0.01686613361040751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,1536,0.016239999731381734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,512,0.015501866738001505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,768,0.015562666455904641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,1024,0.016238933801651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,256,0.015346133708953857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,128,0.015034666657447815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,64,0.01511360009511312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,8192,0.020658133427302043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,32,0.014997333288192749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,10240,0.021387734015782676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,12288,0.021389865875244142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,16384,0.022820266087849934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,65536,0.04511253436406453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,6144,0.020578134059906005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,7168,0.021041067441304524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,5120,0.019310933351516724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,4096,0.018497065703074137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,3072,0.017685333887736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,3584,0.018241065740585326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,2048,0.01665066679318746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,2560,0.017332265774408974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,1536,0.01648319959640503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,128,0.015033599734306336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,768,0.015735466281572977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,1024,0.015983999768892924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,512,0.015299200018246969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,256,0.015280000368754067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,64,0.015053866306940713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,32,0.015198933084805808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,7168,0.02094399929046631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,10240,0.021061333020528157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,16384,0.022637865940729775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,8192,0.020645334323247274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,12288,0.021163733800252278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,65536,0.03886933326721191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,5120,0.019522132476170857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,6144,0.02039573391278585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,4096,0.018143999576568603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,3584,0.018118399381637573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,3072,0.017515732844670614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,2560,0.01702293356259664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,1536,0.0162581334511439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,2048,0.016706132888793947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,768,0.015497600038846334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,512,0.015524267156918844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,1024,0.01618666648864746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,256,0.015262933572133383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,64,0.015026133259137472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,128,0.01495039959748586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,32,0.0149536003669103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,16384,0.022338134050369263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,10240,0.02127679983774821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,6144,0.020190932353337607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,12288,0.021386667092641195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,8192,0.020814933379491172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,7168,0.020617600282033285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,65536,0.03671893278757731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,3584,0.018023467063903807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,5120,0.019291732708613077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,4096,0.018541866540908815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,2560,0.01715839902559916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,3072,0.01770240068435669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,1536,0.016080000003178916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,2048,0.016658133268356322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,1024,0.015941333770751954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,768,0.015689599514007568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,512,0.01562346617380778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,256,0.015280000368754067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,64,0.014889599879582724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,128,0.015154133240381876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,32,0.01497066617012024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,10240,0.021197867393493653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,16384,0.022634667158126832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,12288,0.021347200870513915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,65536,0.036518398920694986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,8192,0.02018773357073466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,5120,0.01924906571706136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,7168,0.020680532852808634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,6144,0.02040106654167175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,4096,0.01842133402824402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,2048,0.016806399822235106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,3072,0.017621332406997682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,2560,0.01700053413709005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,3584,0.018132267395655315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,1536,0.01641493340333303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,1024,0.01606613298257192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,512,0.015598932902018229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,768,0.015799466768900552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,256,0.015075199802716575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,128,0.01471573313077291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,64,0.014938666423161825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,32,0.015081600348154704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,5120,0.13939305941263835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,6144,0.16223360697428385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,7168,0.18652373949686687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,8192,0.20799253781636556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,10240,0.25554666519165037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,4096,0.11326613426208496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,2560,0.08031360308329263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,3584,0.10451306502024334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,3072,0.0918282667795817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,12288,0.30178559621175133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,256,0.025040000677108765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,2048,0.06750079790751139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,512,0.030798933903376263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,1024,0.04453546603520711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,128,0.021313067277272543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,768,0.039241600036621097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,1536,0.05704640150070191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,64,0.01980479955673218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,32,0.01964906652768453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,16384,0.3916223843892416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,7168,0.05765440066655477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,8192,0.06382186810175577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,12288,0.08740693728129069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,10240,0.07528533140818278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,16384,0.10918506781260173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,6144,0.057777067025502525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,3072,0.03472213347752889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,4096,0.040598400433858234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,5120,0.04623466730117798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,2560,0.03159679969151814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,3584,0.03774826526641846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,512,0.018653867642084758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,768,0.020153599977493285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,1024,0.021630932887395225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,256,0.016520532965660095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,1536,0.024650667111078897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,2048,0.028125866254170732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,128,0.01611199975013733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,64,0.01628159979979197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,32,0.016420267025629678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,10240,0.06215039889017741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,7168,0.04965333143870036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,8192,0.054730665683746335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,12288,0.0741706689198812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,16384,0.09275840123494467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,6144,0.04441599845886231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,5120,0.04015359878540039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,4096,0.035892268021901444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,3584,0.03302719990412394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,2560,0.02732906738917033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,3072,0.030923734108606975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,2048,0.025065600872039795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,1536,0.022780799865722658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,768,0.019373865922292073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,1024,0.020637865861256918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,512,0.01658560037612915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,64,0.01630826691786448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,256,0.016244266430536905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,128,0.015658666690190635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,32,0.01614293356736501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,65536,0.3803882598876953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,10240,0.05449493328730265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,8192,0.04667199850082397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,12288,0.06253973245620728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,16384,0.07670079867045085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,7168,0.04729066689809163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,4096,0.03173973361651103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,3072,0.027822933594385785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,6144,0.039662933349609374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,5120,0.03548693259557088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,3584,0.029594665765762328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,2048,0.023384533325831094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,2560,0.025382399559020996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,65536,0.310101318359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,768,0.01859626571337382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,1536,0.021524266401926676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,1024,0.019590399662653604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,512,0.016716800133387246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,256,0.016239999731381734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,64,0.01599679986635844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,32,0.01579839984575907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,128,0.01591253379980723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,10240,0.046809601783752444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,8192,0.04057173331578572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,7168,0.03786666790644328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,12288,0.05322986841201782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,16384,0.06471680005391439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,3584,0.026204800605773924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,3072,0.024605866273244223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,6144,0.03487573464711507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,4096,0.02786453366279602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,5120,0.03110186656316121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,65536,0.2728682518005371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,2048,0.021372799078623453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,2560,0.02291626731554667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,1536,0.020145066579182944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,1024,0.018441599607467652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,768,0.01632960041364034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,512,0.015999999642372132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,256,0.01578879952430725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,128,0.015779200196266174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,64,0.015998933712641397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,32,0.0161461333433787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,8192,0.03796693483988444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,10240,0.04337386687596639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,12288,0.049252267678578696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,7168,0.03536320130030314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,16384,0.05916266838709513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,4096,0.026237867275873822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,65536,0.21993600527445473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,3584,0.024642133712768556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,6144,0.03261546691258748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,5120,0.02950719992319743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,2560,0.02220266660054525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,3072,0.023564799626668295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,2048,0.02155733307202657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,512,0.016244266430536905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,1536,0.019565866390864054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,1024,0.017429333925247193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,768,0.016613333423932394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,256,0.015754666924476624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,64,0.015777066349983215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,128,0.01570026675860087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,32,0.01594986617565155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,10240,0.03965973456700643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,12288,0.044888532161712645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,7168,0.03288106719652812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,16384,0.053787732124328615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,8192,0.03507839838663737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,65536,0.20686400731404625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,6144,0.0300053338209788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,3584,0.023523199558258056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,4096,0.025225599606831867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,5120,0.02720106641451518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,3072,0.022639999787012734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,2560,0.02236586610476176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,2048,0.0202346662680308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,1536,0.019312000274658202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,128,0.015435733397801719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,1024,0.017083734273910522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,768,0.01635840038458506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,256,0.01558080017566681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,512,0.015893333156903586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,32,0.015464533368746439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,64,0.015557333827018738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,8192,0.032295467456181844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,65536,0.17778347333272299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,12288,0.0404853343963623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,10240,0.03609386682510376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,7168,0.030032000939051312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,16384,0.04846719900767009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,6144,0.02760319908459981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,5120,0.025101866324742633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,4096,0.02313279906908671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,3072,0.02136533260345459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,3584,0.02222933371861776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,2560,0.020444800456364952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,2048,0.019349332650502524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,1536,0.018125865856806436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,1024,0.016107733050982155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,768,0.01642346680164337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,512,0.015963733196258545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,128,0.015333333611488342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,64,0.015527466932932535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,256,0.015643733739852905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,32,0.015265066425005594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,65536,0.1537610689798991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,10240,0.03307626644770305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,7168,0.02684906721115112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,8192,0.029113600651423138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,12288,0.03637760082880656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,16384,0.04769173463185628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,5120,0.023568000396092734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,6144,0.02534186641375224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,4096,0.021973333756128945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,3072,0.02055466572443644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,3584,0.021303466955820718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,2048,0.018582399686177573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,2560,0.01965973377227783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,1024,0.016243199507395424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,1536,0.017242666085561117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,768,0.016217600305875143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,256,0.015494400262832641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,512,0.01602133313814799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,128,0.015256533026695251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,64,0.01548373301823934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,32,0.015753600001335143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,65536,0.13096853097279865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,16384,0.04461546738942464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,8192,0.026999467611312868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,12288,0.034371201197306314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,6144,0.02442453304926554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,7168,0.02555626630783081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,10240,0.030836266279220582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,5120,0.022765866915384927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,4096,0.021284266312917074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,3072,0.020088533560434975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,3584,0.020706133047739664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,2560,0.01909760038057963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,1536,0.01684266726175944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,2048,0.01727893352508545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,1024,0.016196266810099284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,128,0.015347199638684592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,768,0.015958399573961893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,512,0.01586026648680369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,256,0.01548373301823934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,65536,0.11910826365152996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,64,0.015273599823315939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,32,0.015134933590888976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,8192,0.02924586733182271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,7168,0.024399999777475992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,10240,0.03236266573270162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,12288,0.0352288007736206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,16384,0.041058135032653806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,6144,0.023143466313680014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,5120,0.022028799851735434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,1536,0.016402133305867515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,3584,0.020196266969045005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,4096,0.02077546715736389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,3072,0.019594667355219524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,2560,0.018386133511861167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,2048,0.01774719953536987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,1024,0.016219733158747356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,128,0.015305599570274353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,64,0.015192533532778421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,768,0.01609599987665812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,256,0.015476266543070475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,512,0.01562773287296295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,32,0.01534293293952942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,65536,0.10858346621195476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,7168,0.023591466744740806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,12288,0.03245226740837097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,10240,0.02722346584002177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,8192,0.024482132991154988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,6144,0.022361600399017335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,16384,0.0374506672223409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,5120,0.021436800559361778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,2560,0.01756160060564677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,4096,0.020196266969045005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,3584,0.019953066110610963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,3072,0.0183786670366923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,2048,0.01712533235549927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,1024,0.016338133811950685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,1536,0.016847999890645345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,768,0.016145066420237223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,512,0.016042666633923848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,256,0.015429332852363586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,64,0.015400532881418863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,128,0.015105066696802774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,65536,0.09833172957102457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,32,0.015105066696802774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,10240,0.027306665976842243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,8192,0.02412373423576355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,7168,0.023162666956583658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,6144,0.02177600065867106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,12288,0.029765333731969195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,16384,0.03418346643447876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,3072,0.018329600493113198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,5120,0.020875734090805054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,4096,0.019742933909098308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,2048,0.017067732413609822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,2560,0.0172160009543101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,3584,0.01918826699256897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,1536,0.016581333676973977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,65536,0.08326186339060465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,1024,0.016237866878509522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,768,0.015939199924468996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,512,0.015787733594576518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,64,0.01527466674645742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,256,0.015362133582433065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,128,0.015301332871119181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,32,0.015254400173823037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,10240,0.02532800038655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,16384,0.02983679970105489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,12288,0.026570665836334228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,7168,0.02403306762377421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,8192,0.023899734020233154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,5120,0.020470400651295982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,6144,0.022974934180577597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,4096,0.018310399850209554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,3584,0.01844053268432617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,2048,0.016915200153986613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,3072,0.017796266078948974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,2560,0.01723626653353373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,65536,0.07246399720509847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,1536,0.016616533199946083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,1024,0.01609493295351664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,512,0.015650133291880287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,768,0.016074666380882265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,256,0.015470932920773825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,64,0.01529813309510549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,128,0.015064533551534018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,32,0.01544426679611206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,16384,0.027063467105229694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,10240,0.02431466579437256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,12288,0.026054400205612182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,8192,0.022476800282796226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,7168,0.021908267339070638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,5120,0.019720532496770225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,6144,0.02104426622390747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,65536,0.060616532961527504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,3072,0.0178656001885732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,4096,0.018666666746139527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,3584,0.018236800034840902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,2048,0.016857600212097167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,2560,0.01739413340886434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,1536,0.016754132509231568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,768,0.016101333498954772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,1024,0.01599253316720327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,512,0.01565439999103546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,128,0.015152000387509666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,256,0.015562666455904641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,32,0.015356799960136414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,64,0.015049599607785544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,10240,0.02333866755167643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,12288,0.0248799999554952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,16384,0.027130667368570966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,8192,0.021144533157348634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,5120,0.019093332688013713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,6144,0.02091946601867676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,7168,0.020889600118001304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,3584,0.01795519987742106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,65536,0.055034665266672764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,4096,0.0185973326365153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,2560,0.017108267545700072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,3072,0.017720532417297364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,2048,0.016676266988118492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,1536,0.01641706625620524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,256,0.01515733301639557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,1024,0.01588053305943807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,512,0.015636266271273295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,768,0.01607146660486857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,128,0.015148799618085226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,64,0.015057067076365152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,32,0.015296000242233276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,10240,0.02258239984512329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,16384,0.025033599138259886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,12288,0.024246400594711302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,7168,0.02095253268877665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,65536,0.04912319978078206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,6144,0.0204586664835612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,8192,0.02077440023422241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,5120,0.019186133146286012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,4096,0.018161066373189292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,3584,0.017806933323542277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,2560,0.01713599960009257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,3072,0.01737706661224365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,1024,0.015997866789499916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,1536,0.01638826628526052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,2048,0.016707199811935424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,512,0.015432533621788026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,768,0.015490133563677469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,64,0.015128533045450846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,256,0.01524906655152639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,128,0.01499626636505127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,32,0.015225600202878317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,16384,0.02297280033429464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,12288,0.020891733964284263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,65536,0.039622398217519124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,10240,0.020862932999928793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,8192,0.020240000883738198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,7168,0.020642133553822835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,6144,0.0202346662680308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,5120,0.01906879941622416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,3072,0.017502933740615845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,4096,0.018210132916768394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,3584,0.01798293391863505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,2560,0.017029333114624023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,768,0.015691733360290526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,2048,0.01660480002562205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,1024,0.016033066312472026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,1536,0.01639573375384013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,512,0.015285332997639975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,128,0.01502293348312378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,256,0.015253333250681558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,64,0.015079466501871744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,32,0.014998400211334228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,12288,0.02081813414891561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,10240,0.02101226647694906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,16384,0.02220373352368673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,65536,0.036848000685373944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,8192,0.02047146757443746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,7168,0.020518400271733604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,4096,0.01824959913889567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,5120,0.01919999917348226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,6144,0.020182400941848755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,3584,0.018118399381637573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,2048,0.016597333550453185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,3072,0.01753600041071574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,2560,0.017151999473571777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,1536,0.01646080017089844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,1024,0.015862400333086647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,256,0.015196800231933594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,768,0.015561599532763162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,64,0.01504533290863037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,512,0.015496533115704855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,128,0.01511146624883016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,32,0.014940800269444785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,16384,0.021869866053263347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,10240,0.021052799622217813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,7168,0.02070186734199524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,12288,0.020750933885574342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,65536,0.03387093146642049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,8192,0.020268799861272176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,6144,0.020134399334589638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,5120,0.019157334168752035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,4096,0.018347734212875368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,3584,0.01797440052032471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,2560,0.016832000017166136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,3072,0.01737706661224365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,1024,0.015998933712641397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,2048,0.016734933853149413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,1536,0.01604373355706533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,64,0.015049599607785544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,768,0.0156960000594457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,512,0.015379200379053751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,256,0.015046399831771851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,128,0.014918399850527444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,32,0.015093333522478738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,16384,0.02190399964650472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,65536,0.03518079916636149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,8192,0.02003093361854553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,10240,0.020873600244522096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,12288,0.02076373298962911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,7168,0.02022613286972046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,6144,0.020194133122762047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,5120,0.019352533419926963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,4096,0.018145066499710084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,3584,0.01769066651662191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,3072,0.017451733350753784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,2560,0.017087999979654947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,1024,0.01588586668173472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,2048,0.0165994664033254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,1536,0.01627946694691976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,768,0.01569493313630422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,512,0.015290666619936624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,64,0.014963199694951376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,256,0.015170133113861084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,128,0.015030399958292643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,32,0.014879999558130899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,5120,0.1366528034210205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,6144,0.16047360102335612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,7168,0.18484907150268554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,8192,0.20592853228251137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,10240,0.25329920450846355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,12288,0.29849173227945963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,3584,0.10252799987792968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,4096,0.11187307039896648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,2560,0.0782528003056844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,3072,0.0909056027730306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,256,0.022746666272481283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,512,0.03014933268229167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,2048,0.06542933384577433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,768,0.037539199988047285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,64,0.018308266003926595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,128,0.020413867632548013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,1024,0.04294613202412923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,1536,0.06137813329696655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,32,0.018614399433135986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,16384,0.3958944002787272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,8192,0.06593386729558309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,10240,0.07576426664988199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,7168,0.0588266650835673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,12288,0.08780906995137533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,6144,0.05792640050252279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,16384,0.11279786427815755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,5120,0.047926398118336995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,4096,0.04094506502151489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,3584,0.03753493229548137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,1536,0.024387200673421226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,3072,0.03410559892654419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,2560,0.031112533807754517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,2048,0.0276853342851003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,512,0.018474666277567546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,768,0.019994666179021202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,1024,0.021513599157333373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,256,0.01628373364607493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,128,0.016090666254361473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,32,0.016192000110944113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,64,0.01614400049050649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,7168,0.04714986483256022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,8192,0.05161173343658447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,10240,0.06011946598688761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,12288,0.06912960211435953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,16384,0.0856223980585734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,5120,0.03846933444341023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,6144,0.04286400079727173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,3584,0.03146666685740153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,4096,0.0335754672686259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,2560,0.02664639949798584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,3072,0.029285333553949994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,2048,0.023940267165501912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,1536,0.02202346722284953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,1024,0.019912532965342202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,768,0.018794665733973183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,256,0.01611199975013733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,512,0.016204800208409628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,128,0.015808000167210897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,64,0.01595626672108968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,32,0.015811199943224587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,8192,0.04563306570053101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,10240,0.05312000115712484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,12288,0.06075626611709595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,65536,0.4030144055684407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,16384,0.07447786331176758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,7168,0.04205013513565063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,5120,0.035028266906738284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,6144,0.03850133419036865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,3584,0.02911146680514018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,3072,0.026935466130574542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,4096,0.03346560001373291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,1024,0.019385600090026857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,2560,0.02472106615702311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,1536,0.021348265806833903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,65536,0.3028010686238607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,2048,0.023028266429901124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,768,0.01770240068435669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,512,0.01598186691602071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,256,0.015708800156911215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,128,0.015491200486818948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,64,0.01516800026098887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,32,0.01572160025437673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,8192,0.03998719851175944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,7168,0.0371509313583374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,16384,0.0638592004776001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,10240,0.04628373384475708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,12288,0.05240213473637899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,4096,0.02762346665064494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,3584,0.025700267155965167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,3072,0.024150399367014568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,65536,0.27171198527018225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,6144,0.03389546473821004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,5120,0.03127040068308513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,2560,0.022784000635147093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,2048,0.021314134200414024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,1536,0.019939200083414713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,1024,0.019022933642069497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,768,0.016420267025629678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,256,0.015779200196266174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,512,0.01628266672293345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,32,0.015862400333086647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,64,0.015752533078193666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,128,0.015703466534614564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,7168,0.03505493402481079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,10240,0.04284799893697103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,8192,0.03807146549224853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,12288,0.04853866497675578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,16384,0.059180800120035806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,65536,0.22690134048461913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,4096,0.02643199960390727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,3072,0.02337813377380371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,3584,0.024844799439112344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,5120,0.029513599475224812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,6144,0.03237333297729492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,1536,0.019700266917546592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,2048,0.020947200059890748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,2560,0.02318613330523173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,1024,0.017422932386398315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,768,0.016241066654523215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,256,0.015641599893569946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,512,0.016110933820406594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,128,0.015611732999483744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,64,0.01553813318411509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,32,0.015715199708938598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,16384,0.05284693241119385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,12288,0.044326400756835936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,10240,0.039338668187459305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,7168,0.03217599987983703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,8192,0.03431679805119832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,65536,0.20077120463053383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,6144,0.029727999369303388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,4096,0.024771199623743693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,5120,0.027104000250498455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,3584,0.023471999168395995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,3072,0.022639999787012734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,2560,0.02223466634750366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,2048,0.020284799734751384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,1536,0.01916266679763794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,1024,0.016309332847595216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,768,0.016345600287119545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,128,0.015413332978884378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,512,0.015995732943216958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,256,0.015496533115704855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,64,0.015503999590873719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,32,0.015614933768908181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,65536,0.1784981409708659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,10240,0.03548906644185384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,12288,0.03999360005060832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,16384,0.04769173463185628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,8192,0.03156266609827678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,7168,0.02956693371136983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,3584,0.02218666672706604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,5120,0.024845866362253825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,6144,0.027270400524139406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,4096,0.02291839917500814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,3072,0.021204266945521036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,2560,0.020551466941833497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,2048,0.01950613260269165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,1536,0.01821546753247579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,768,0.015958399573961893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,1024,0.01600106656551361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,512,0.015827199816703795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,128,0.015454933047294617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,256,0.015625600020090738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,64,0.015233066678047181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,32,0.015397333105405173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,12288,0.035751465956370035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,65536,0.14796800613403321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,10240,0.03232746720314026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,16384,0.045627733071645096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,7168,0.026919466257095338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,6144,0.025095466772715253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,8192,0.028617600599924724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,5120,0.023269333442052207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,2560,0.019657599925994872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,3584,0.020938666661580403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,4096,0.021925334135691324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,3072,0.02039573391278585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,2048,0.018568533658981323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,1024,0.01606613298257192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,1536,0.016679465770721436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,768,0.01611199975013733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,64,0.015242666999499003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,512,0.01575040022532145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,128,0.015092266599337259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,256,0.01534933348496755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,32,0.015398400028546652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,65536,0.12511253356933594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,8192,0.026787199576695758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,6144,0.024014933904012045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,10240,0.030330665906270343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,12288,0.03743360042572021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,7168,0.025116799275080363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,16384,0.04284799893697103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,4096,0.02139520049095154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,5120,0.02253226637840271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,3584,0.020602667331695558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,2048,0.01739413340886434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,3072,0.019849600394566853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,2560,0.01918720006942749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,1536,0.016165332992871602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,1024,0.01616106629371643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,768,0.015740799903869628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,512,0.015603199601173401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,256,0.015521066387494406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,128,0.015304533640543619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,64,0.015441067020098367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,65536,0.11346773306528728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,32,0.01529706617196401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,8192,0.02540480097134908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,10240,0.028014934062957762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,16384,0.03938773473103841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,7168,0.02444266676902771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,12288,0.03151786724726359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,5120,0.021808000405629475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,6144,0.02285439968109131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,4096,0.020897066593170165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,2048,0.01783999999364217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,3584,0.020002132654190062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,3072,0.019444266955057778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,2560,0.018351999918619792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,1536,0.016782933473587038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,768,0.015997866789499916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,1024,0.016376533110936484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,512,0.015688533584276833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,256,0.015433599551518759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,65536,0.10330560207366943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,128,0.015233066678047181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,64,0.015092266599337259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,32,0.01527253290017446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,12288,0.02909440000851949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,6144,0.02203413248062134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,10240,0.02616106669108073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,8192,0.024324266115824382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,7168,0.023371734221776328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,16384,0.03635733524958293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,4096,0.01994346578915914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,5120,0.021384533246358237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,3584,0.01953173279762268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,2048,0.017064533631006875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,3072,0.01862293283144633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,2560,0.01727893352508545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,1536,0.01663253307342529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,1024,0.01627626617749532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,768,0.015707733233769734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,512,0.015640532970428465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,65536,0.08986132939656576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,256,0.015432533621788026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,64,0.015229866902033488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,128,0.015011200308799743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,32,0.015220266580581666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,8192,0.02476373314857483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,10240,0.026685865720113118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,12288,0.028938666979471846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,16384,0.03294080098470052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,7168,0.02413439949353536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,6144,0.021919999519983926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,5120,0.020466132958730062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,3584,0.01875306765238444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,4096,0.01965120037396749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,3072,0.017518933614095053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,1536,0.016525866587956746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,2560,0.016755199432373045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,2048,0.016731733083724977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,65536,0.07864426771799723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,1024,0.01619733373324076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,768,0.01592639982700348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,256,0.015314132968584696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,512,0.01567893326282501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,128,0.015014400084813436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,64,0.015103999773661295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,32,0.015015467007954916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,10240,0.024809600909550984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,12288,0.025737599531809492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,16384,0.02932586669921875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,7168,0.021681066354115805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,8192,0.023570134242375692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,5120,0.019373865922292073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,6144,0.02131519913673401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,4096,0.01830186645189921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,3072,0.01755839983622233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,3584,0.01767146587371826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,2560,0.016989866892496742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,2048,0.016535466909408568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,65536,0.06818133195241292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,1024,0.01597546637058258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,1536,0.01628159979979197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,768,0.015847466389338174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,512,0.015726932883262636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,256,0.015223466356595359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,128,0.015043200055758158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,32,0.015123200416564942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,64,0.015195733308792115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,16384,0.026150399446487428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,8192,0.02245546579360962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,7168,0.02164586583773295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,12288,0.02474666635195414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,10240,0.023614933093388878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,6144,0.020045866568883262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,65536,0.05655466715494791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,5120,0.019179733594258626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,3584,0.01802560091018677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,4096,0.018377600113550822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,3072,0.017486933867136636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,2560,0.017016534010569254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,2048,0.017051732540130614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,1536,0.016313599546750386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,768,0.015735466281572977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,512,0.015589333573977151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,1024,0.015995732943216958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,256,0.015422933300336204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,128,0.015160533785820007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,16384,0.025231999158859254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,32,0.014986667037010192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,64,0.015424000223477683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,12288,0.02318293253580729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,7168,0.02083946665128072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,10240,0.022051199277242025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,8192,0.020220800240834554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,6144,0.019832533597946168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,5120,0.019337600469589232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,65536,0.05096640189488729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,4096,0.018407466014226277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,3584,0.017753599087397258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,3072,0.01758613387743632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,2048,0.016502400239308677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,2560,0.016962132851282754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,512,0.015455999970436096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,1536,0.01652906636397044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,1024,0.015921066204706825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,768,0.015794133146603904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,32,0.01497066617012024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,256,0.015282133221626281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,128,0.014989866813023885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,64,0.015031466881434122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,12288,0.0204586664835612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,16384,0.023244800170262654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,10240,0.02119040091832479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,7168,0.020359466473261513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,8192,0.02040106654167175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,65536,0.044846932093302413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,5120,0.019019732872645058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,6144,0.020012799898783365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,3584,0.017783466974894205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,4096,0.01844373345375061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,3072,0.017352533340454102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,768,0.015677866339683533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,2560,0.017114667097727458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,1536,0.016432000199953715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,2048,0.016823466618855795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,512,0.015545599659283958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,1024,0.015821866194407144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,128,0.015011200308799743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,256,0.015006933609644571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,32,0.014870400230089823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,64,0.015194666385650635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,16384,0.021498666206995646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,12288,0.02071466644605001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,10240,0.020728532473246256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,8192,0.02005973259607951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,65536,0.03549439907073974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,7168,0.0204586664835612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,6144,0.020231467485427857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,3584,0.017554134130477905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,5120,0.01911999980608622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,4096,0.01803306738535563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,3072,0.017233065764109292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,2560,0.017110399405161538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,1024,0.015754666924476624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,2048,0.01665066679318746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,768,0.015715199708938598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,1536,0.016522666811943053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,256,0.014961066842079162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,64,0.014971733093261719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,512,0.015424000223477683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,128,0.015058133006095886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,32,0.014998400211334228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,10240,0.02084160049756368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,16384,0.02140799959500631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,12288,0.0203658660252889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,65536,0.03243093291918437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,8192,0.02031466762224833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,7168,0.020539732774098714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,6144,0.019939200083414713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,5120,0.019058134158452353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,4096,0.018369066715240478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,3584,0.01758079926172892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,3072,0.017525333166122436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,1024,0.015889066457748412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,2560,0.016977065801620485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,1536,0.016327466567357382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,2048,0.016724266608556113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,512,0.01527679959932963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,256,0.015171200037002563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,768,0.015718400478363037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,128,0.015078399578730264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,32,0.015109333395957946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,64,0.014903466900189719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,8192,0.01997440059979757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,12288,0.020402133464813232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,65536,0.030854399998982745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,16384,0.020992000897725425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,10240,0.020491733153661092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,7168,0.02031893332799276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,6144,0.019742933909098308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,5120,0.019016534090042114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,4096,0.018206934134165444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,3072,0.017166932423909508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,3584,0.017942400773366292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,2560,0.016945066054662068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,2048,0.016618667046229045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,1024,0.015754666924476624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,1536,0.01636373301347097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,768,0.01569066643714905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,512,0.015217066804567973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,256,0.015253333250681558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,32,0.014920533696810404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,128,0.015015467007954916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,64,0.01509119967619578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,12288,0.020308266083399452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,8192,0.019844265778859456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,16384,0.021126399437586464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,10240,0.020490666230519615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,65536,0.02906559904416402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,7168,0.02024959921836853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,6144,0.01975040038426717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,4096,0.018182400862375894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,5120,0.01880106727282206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,3584,0.017638399203618368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,3072,0.01736746629079183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,1536,0.01617173353830973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,2560,0.016781866550445557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,2048,0.01651306649049123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,1024,0.015681067109107973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,768,0.015614933768908181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,512,0.015364266435305276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,128,0.014994133512179056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,256,0.015194666385650635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,64,0.014820266763369241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,32,0.015037866433461508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,5120,0.1320863962173462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,6144,0.1561247984568278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,7168,0.17719999949137372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,8192,0.2006773312886556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,10240,0.24440959294637046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,4096,0.1084330638249715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,3584,0.0990325371424357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,3072,0.08711893558502197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,2560,0.07564373016357422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,12288,0.28978452682495115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,2048,0.0639402667681376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,256,0.023396267493565878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,128,0.019964800278345744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,1536,0.05347946484883627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,512,0.02908160090446472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,768,0.03627520004908244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,32,0.018450133005777993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,1024,0.04201706647872925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,64,0.017619200547536216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,16384,0.3842890739440918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,7168,0.05598400036493937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,6144,0.05601386626561483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,8192,0.06251413424809774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,10240,0.07292053699493409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,12288,0.08464746475219727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,16384,0.10589973131815593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,3072,0.033573333422342935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,4096,0.039214932918548585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,5120,0.044955734411875406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,3584,0.03654719988505046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,2560,0.030817067623138426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,1536,0.024239999055862427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,1024,0.021387734015782676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,768,0.019873066743214925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,2048,0.027449599901835126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,512,0.018304000298182167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,128,0.01601066688696543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,256,0.01621333360671997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,64,0.01592639982700348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,32,0.01599679986635844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,8192,0.05196693340937296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,7168,0.04805866479873657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,10240,0.061559466520945225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,12288,0.06908266544342041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,16384,0.0893887996673584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,6144,0.042819201946258545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,5120,0.03903146584828694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,4096,0.034701867898305254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,3584,0.031624533732732135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,2560,0.026652799050013228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,3072,0.029683200518290202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,1024,0.019859200716018675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,2048,0.024615466594696045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,1536,0.022503467400868733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,256,0.0158869336048762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,768,0.018964266777038573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,512,0.01649493376413981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,128,0.0155349334081014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,64,0.015864533185958863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,32,0.01576533317565918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,8192,0.04554773171742757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,65536,0.3683850606282552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,10240,0.0529802680015564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,12288,0.060344533125559485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,16384,0.07263253529866537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,7168,0.04183573325475057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,4096,0.030509867270787555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,5120,0.03447893460591634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,6144,0.0414463996887207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,3072,0.026578134298324584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,3584,0.028655999898910524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,2560,0.02458239992459615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,65536,0.2971839904785156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,1024,0.019346133867899577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,2048,0.02290346622467041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,1536,0.02109866738319397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,768,0.017412267128626504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,128,0.01564586659272512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,512,0.015960533420244852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,256,0.015833600362141927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,32,0.015713066856066386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,64,0.015779200196266174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,7168,0.03662399848302205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,8192,0.03981546560923259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,10240,0.04573546648025513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,12288,0.05157333215077718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,16384,0.06246933142344156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,3072,0.02418880065282186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,3584,0.025410133600234985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,65536,0.2595061302185059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,4096,0.027560534079869588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,5120,0.03057066599527995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,6144,0.033998934427897136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,2560,0.02284053365389506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,2048,0.021143466234207153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,1024,0.018119466304779053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,256,0.01576959987481435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,1536,0.02100480000178019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,768,0.01663253307342529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,512,0.016203733285268147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,32,0.01573013365268707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,128,0.01548373301823934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,64,0.015783466895421348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,10240,0.042242133617401124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,7168,0.03465066750844319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,8192,0.036950401465098065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,12288,0.04796160062154134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,16384,0.057265067100524904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,6144,0.032102400064468385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,4096,0.025868799289067584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,3584,0.024804266293843587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,65536,0.2082815965016683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,3072,0.023510400454203287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,5120,0.029233066240946452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,2560,0.02214933236440023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,2048,0.020579200983047486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,768,0.015899733702341715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,1536,0.019595734278361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,1024,0.017086933056513466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,512,0.01593706707159678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,256,0.0156768004099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,128,0.015281066298484802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,64,0.015589333573977151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,32,0.015609600146611533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,12288,0.04407573143641154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,65536,0.20108373959859213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,16384,0.05431893269220987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,8192,0.03544640143712362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,7168,0.03303146759668986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,10240,0.03947093486785889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,5120,0.02880000074704488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,6144,0.029740800460179646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,4096,0.024943999449412026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,3584,0.024000000953674317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,3072,0.02252266605695089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,2560,0.02133973240852356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,2048,0.019951999187469482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,1536,0.01888213356335958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,1024,0.017148800690968833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,768,0.01629973351955414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,128,0.015285332997639975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,512,0.015825066963831583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,256,0.015337600310643514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,64,0.015399466951688132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,32,0.015346133708953857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,8192,0.031166932980219525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,65536,0.16825706164042156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,12288,0.039155201117197676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,10240,0.03535039822260539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,7168,0.02913386623064677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,16384,0.046402132511138915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,6144,0.02687893311182658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,3072,0.02132373253504435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,2560,0.020492800076802573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,4096,0.023031467199325563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,3584,0.022258132696151733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,5120,0.024795732895533242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,2048,0.019577600558598838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,1024,0.016109866897265117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,1536,0.01807039976119995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,768,0.016100266575813295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,512,0.015590399503707886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,256,0.015449600418408713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,64,0.015309866269429526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,128,0.015381333231925965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,32,0.015400532881418863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,65536,0.14212586085001627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,7168,0.026292266448338826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,8192,0.02799466649691264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,10240,0.03189546664555867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,12288,0.03497386773427327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,16384,0.04374399979909261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,5120,0.02331839998563131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,3584,0.021157334248224892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,4096,0.02172693411509196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,6144,0.02480640014012655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,2560,0.019375999768575035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,3072,0.020222934087117513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,1536,0.016620799899101257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,1024,0.01626240015029907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,2048,0.0183242658774058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,768,0.015987199544906617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,512,0.015603199601173401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,256,0.015307733416557312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,128,0.015198933084805808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,64,0.015400532881418863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,32,0.015561599532763162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,65536,0.11954560279846191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,8192,0.026293333371480303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,7168,0.02498133381207784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,10240,0.029869866371154786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,12288,0.03303359945615132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,5120,0.022398932774861654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,16384,0.04122346639633179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,6144,0.0236735999584198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,4096,0.021194666624069214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,2560,0.01916586756706238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,3584,0.020565332969029744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,3072,0.019845332702000937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,1024,0.016139733791351318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,1536,0.016244266430536905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,2048,0.016829866170883178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,768,0.015921066204706825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,512,0.01571626663208008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,128,0.015141333142916361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,256,0.015205333630243937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,64,0.015324800213177999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,65536,0.10715200106302898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,32,0.015000533064206442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,16384,0.038840532302856445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,10240,0.03112000028292338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,8192,0.0286901334921519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,6144,0.022711465756098427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,7168,0.023847466707229613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,12288,0.03436160087585449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,5120,0.021513599157333373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,3584,0.019870932896931967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,1536,0.016692266861597697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,4096,0.020248534282048543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,3072,0.019177599747975668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,2560,0.017777067422866822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,2048,0.016671999295552572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,768,0.01576533317565918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,1024,0.016085333625475564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,256,0.015121066570281982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,64,0.015135999520619711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,512,0.015518933534622192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,65536,0.09838186899820964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,128,0.015268266201019287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,32,0.015044266978899637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,6144,0.02193386753400167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,12288,0.030419200658798218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,7168,0.02517333428064982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,8192,0.02625173330307007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,10240,0.028016000986099243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,16384,0.03511679967244466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,5120,0.02099840044975281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,4096,0.019721599419911702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,3584,0.019437867403030395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,3072,0.01845226685206095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,2560,0.016819200913111367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,1536,0.016277333100636802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,2048,0.016873600085576375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,1024,0.016301866372426352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,768,0.0159061332543691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,512,0.01563093364238739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,65536,0.08808106581370036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,128,0.015145599842071533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,256,0.015170133113861084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,64,0.014962133765220643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,32,0.015184000134468079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,10240,0.026210133234659833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,12288,0.028009599447250365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,8192,0.024732800324757893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,16384,0.031982932488123575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,5120,0.020490666230519615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,6144,0.021436800559361778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,7168,0.023692800601323446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,2560,0.01699840029080709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,4096,0.019385600090026857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,3584,0.01816213329633077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,3072,0.017488000790278117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,2048,0.016745599110921223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,1536,0.01618559956550598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,1024,0.016076800227165223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,65536,0.07492372989654542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,768,0.01590720017751058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,256,0.01527466674645742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,128,0.015190399686495461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,512,0.015451733271280924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,32,0.015059199929237366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,64,0.014904533823331198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,7168,0.02152959903081258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,10240,0.024737066030502318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,12288,0.025677865743637084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,16384,0.02845120032628377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,8192,0.022230400641759237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,6144,0.020627200603485107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,5120,0.019479467471440633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,4096,0.018003199497858682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,3584,0.017569067080815633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,65536,0.06402986844380697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,3072,0.01699626644452413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,2560,0.01695466637611389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,1536,0.016229333480199178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,2048,0.016603733102480568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,1024,0.016065067052841185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,768,0.01585813363393148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,256,0.01536853313446045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,64,0.014910933375358582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,512,0.015621333320935567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,128,0.01497066617012024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,32,0.01490133305390676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,16384,0.025788799921671553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,10240,0.02363413373629252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,12288,0.02344320019086202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,8192,0.022533333301544188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,7168,0.021916800737380983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,6144,0.01999680002530416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,4096,0.018158932526906334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,5120,0.01917440096537272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,65536,0.05248213211695353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,3584,0.01802133321762085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,3072,0.01758079926172892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,2560,0.016962132851282754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,1536,0.01618026693662008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,2048,0.016634666919708253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,1024,0.016006400187810264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,768,0.015763200322786965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,64,0.01504746675491333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,512,0.01591679950555166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,256,0.015309866269429526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,128,0.014932266871134438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,32,0.014908799529075622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,16384,0.024174932638804117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,12288,0.023809067408243813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,8192,0.021307732661565146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,10240,0.022655999660491942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,7168,0.02058560053507487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,4096,0.01806933283805847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,65536,0.04646613200505574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,5120,0.018895999590555827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,6144,0.019934932390848793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,2048,0.016691199938456216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,3072,0.017468800147374473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,3584,0.017754666010538735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,2560,0.016809600591659545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,1024,0.01591253379980723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,1536,0.016249600052833556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,768,0.015480533242225647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,512,0.015424000223477683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,128,0.015095466375350952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,256,0.015331199765205384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,12288,0.02118720014890035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,32,0.015069866180419922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,64,0.01504746675491333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,16384,0.02320746580759684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,65536,0.03995946645736694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,7168,0.02044373353322347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,10240,0.020784000555674233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,8192,0.020045866568883262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,6144,0.01999359925587972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,5120,0.01885120073954264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,4096,0.017884800831476845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,3072,0.01735146641731262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,3584,0.017812265952428182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,2560,0.017179733514785765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,1536,0.016331733266512553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,2048,0.01676693360010783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,512,0.015304533640543619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,1024,0.015801599621772765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,768,0.01548373301823934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,256,0.015180800358454385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,64,0.014987732966740927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,128,0.015086932977040609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,32,0.015122133493423461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,65536,0.03407893180847168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,10240,0.020435200134913126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,16384,0.020906666914621987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,12288,0.020075732469558717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,8192,0.019822933276494346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,7168,0.02039360006650289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,6144,0.020121600230534872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,5120,0.01875093380610148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,4096,0.018172800540924072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,3584,0.017514665921529136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,3072,0.017081600427627564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,2560,0.01709973414738973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,2048,0.01676586667696635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,1024,0.01585813363393148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,768,0.015657599767049155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,1536,0.01624853312969208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,512,0.015336533387502035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,128,0.015105066696802774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,256,0.014993066589037577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,32,0.014899200201034546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,64,0.01516480048497518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,16384,0.020890667041142782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,65536,0.030151466528574627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,7168,0.020409599939982096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,10240,0.020555732647577922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,12288,0.020115200678507486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,8192,0.0197269340356191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,6144,0.01993066668510437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,3584,0.017644800742467246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,4096,0.018026665846506754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,5120,0.019064533710479736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,3072,0.01727786660194397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,2560,0.016730666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,2048,0.01646506687005361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,1024,0.015737600127855935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,1536,0.016356266538302102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,768,0.01562346617380778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,256,0.015030399958292643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,512,0.015539200107256571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,32,0.014863999684651694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,128,0.014841600259145101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,64,0.014749866724014283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,8192,0.01972480018933614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,12288,0.020137600104014077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,16384,0.020667733748753865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,65536,0.02757333318392436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,10240,0.020295466979344687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,6144,0.01962239940961202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,7168,0.020297600825627645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,5120,0.01877760092417399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,4096,0.01797013282775879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,3584,0.017576533555984496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,3072,0.017384533087412515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,2560,0.016986666123072307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,1024,0.015740799903869628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,2048,0.016427733500798545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,768,0.01572160025437673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,1536,0.016037333011627197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,512,0.015333333611488342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,64,0.014893866578737893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,256,0.015244799852371215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,128,0.01502826710542043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,32,0.01479039986928304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,12288,0.020082134008407592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,16384,0.02057386636734009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,10240,0.02008533279101054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,65536,0.026339199145634967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,8192,0.019732266664505005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,7168,0.020113066832224528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,3584,0.0175872008005778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,6144,0.01962239940961202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,5120,0.01874026656150818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,3072,0.01722986698150635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,4096,0.017937066157658894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,2560,0.016908800601959227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,1536,0.01611199975013733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,2048,0.016774400075276693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,768,0.015496533115704855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,1024,0.015595733126004537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,512,0.015448533495267234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,64,0.014760532975196838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,256,0.014914133151372275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,128,0.014946132898330688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,32,0.014997333288192749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,5120,0.13027413686116535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,6144,0.1528341293334961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,7168,0.17661226590474446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,8192,0.1994922637939453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,10240,0.2420501391092936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,2560,0.0745141347249349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,4096,0.10732906659444172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,3584,0.09701440334320069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,3072,0.08642346858978271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,12288,0.28774827321370444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,1536,0.05235946575800578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,1024,0.04072106679280599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,768,0.035335465272267656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,2048,0.06291413307189941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,256,0.023002666234970093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,512,0.028385066986083986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,128,0.01948266625404358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,64,0.017847466468811034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,32,0.018029866615931193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,16384,0.3800554593404134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,7168,0.055446398258209226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,10240,0.07215999762217204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,8192,0.06100373268127442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,12288,0.08413013617197672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,6144,0.05570666790008545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,16384,0.10486506621042888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,4096,0.038737066586812335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,3584,0.03617279926935832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,3072,0.03304319977760315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,5120,0.04441706736882527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,2560,0.030347732702891032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,1536,0.024146133661270143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,2048,0.02696320017178853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,1024,0.02113920052846273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,512,0.01811199982961019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,768,0.019730132818222047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,128,0.015685333808263143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,256,0.016129066546758018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,64,0.015658666690190635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,32,0.016078933080037435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,10240,0.05942720174789429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,12288,0.06849706967671712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,7168,0.0476032018661499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,8192,0.052485334873199466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,16384,0.0888970692952474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,5120,0.037674665451049805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,6144,0.04193919897079468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,4096,0.03425066471099854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,3584,0.031541333595911665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,3072,0.0293503999710083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,2560,0.026553599039713542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,2048,0.024013866980870567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,1536,0.022256000836690267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,1024,0.01972480018933614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,512,0.01605013310909271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,768,0.018609066804250084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,256,0.015844266613324484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,128,0.015620266397794088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,64,0.015378133455912272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,32,0.015482667088508605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,65536,0.3654143969217936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,8192,0.04492586851119995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,10240,0.05214720169703165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,12288,0.05973333517710368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,16384,0.0734656016031901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,6144,0.037596801916758224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,7168,0.04500266710917155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,5120,0.03382720152537028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,3584,0.028604799509048463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,4096,0.030186667044957476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,3072,0.026273065805435182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,2560,0.024502400557200113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,65536,0.29061225255330403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,2048,0.022501333554585775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,1536,0.020856533447901407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,1024,0.01926079988479614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,768,0.017826133966445924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,512,0.01602453291416168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,64,0.015507200360298156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,128,0.015264000495274863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,32,0.01567039986451467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,256,0.01516480048497518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,7168,0.035869868596394856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,12288,0.050929065545399985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,8192,0.03913493156433105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,10240,0.04458026488622029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,16384,0.06146666606267294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,5120,0.030136533578236896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,3072,0.02395840088526408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,3584,0.02525866627693176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,6144,0.03323839902877808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,4096,0.027058132489522296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,65536,0.25252159436543786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,2560,0.022605866193771362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,2048,0.02099626660346985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,1536,0.019792000452677407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,256,0.015682133038838704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,1024,0.017678932348887125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,768,0.016424533724784852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,512,0.015404799580574035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,32,0.015656532843907674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,128,0.015365333358446757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,64,0.015286399920781454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,8192,0.03639573256174723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,10240,0.04151893456776937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,65536,0.20148266156514488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,12288,0.047178665796915695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,7168,0.03381653229395549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,16384,0.056493866443634036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,4096,0.02754773298899333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,3584,0.02449173331260681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,3072,0.023069866498311362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,2560,0.021939200162887574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,5120,0.028541866938273115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,6144,0.03146666685740153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,2048,0.020590933163960774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,128,0.01528320014476776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,256,0.015599999825159708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,1024,0.01686506668726603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,1536,0.019320533672968546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,512,0.015637333194414772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,768,0.01579200029373169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,64,0.015198933084805808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,32,0.015390933553377787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,8192,0.03408000071843465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,10240,0.03887253204981486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,12288,0.043305599689483644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,7168,0.032459733883539836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,65536,0.18976106643676757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,16384,0.05332373380661011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,4096,0.024567466974258424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,5120,0.027334400018056232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,6144,0.029525333642959596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,3584,0.023763199647267662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,3072,0.023923200368881226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,2560,0.021271467208862305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,2048,0.020163200298945107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,1536,0.018948266903559365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,1024,0.01578133304913839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,768,0.015873066584269204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,512,0.015786666671435037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,128,0.015280000368754067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,256,0.015288533767064414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,32,0.015125333269437154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,64,0.01548906664053599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,8192,0.030836266279220582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,12288,0.03863893349965413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,65536,0.15906666119893392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,16384,0.0455562671025594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,7168,0.0287882665793101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,10240,0.03479466835657756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,6144,0.026627200841903686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,5120,0.024523733059565227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,3072,0.021268266439437866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,4096,0.022807466983795165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,3584,0.02217280069986979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,2560,0.020122667153676353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,2048,0.019309866428375243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,1536,0.017398399114608765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,1024,0.016219733158747356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,128,0.015133866667747497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,768,0.015829333662986757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,512,0.01575040022532145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,256,0.015255467096964518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,64,0.015244799852371215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,32,0.01539306640625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,65536,0.13817599614461262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,8192,0.027560534079869588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,10240,0.031224532922108965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,12288,0.03463360071182251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,7168,0.02595733404159546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,16384,0.04241280158360799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,6144,0.02436800003051758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,5120,0.02299840052922567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,4096,0.021653334299723305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,3072,0.020196266969045005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,1536,0.016520532965660095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,3584,0.02108373244603475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,2560,0.01924906571706136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,2048,0.018578133980433145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,512,0.015660799543062844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,768,0.01585706671079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,1024,0.01588053305943807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,256,0.015384533007939658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,64,0.015447466572125753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,128,0.01498240033785502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,32,0.015470932920773825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,65536,0.11535147031148274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,7168,0.02476693391799927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,8192,0.025991467634836833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,6144,0.023569067319234215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,10240,0.031548800071080525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,12288,0.032305065790812174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,16384,0.03993599812189738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,5120,0.022165334224700926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,3584,0.020364799102147422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,4096,0.02076479991277059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,3072,0.019679999351501463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,2560,0.019038933515548705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,2048,0.017119999726613364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,1024,0.01602453291416168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,1536,0.016745599110921223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,512,0.015481600165367126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,768,0.01572160025437673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,256,0.015336533387502035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,64,0.015204266707102457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,32,0.015228799978892007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,128,0.014994133512179056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,65536,0.10269653002421061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,10240,0.030136533578236896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,8192,0.02762773235638936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,6144,0.022678399085998537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,7168,0.02367573380470276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,16384,0.038209064801534014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,12288,0.03307733337084452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,5120,0.021654399236043294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,4096,0.020143999656041463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,3584,0.01978879968325297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,3072,0.019256534179051717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,2560,0.017924267053604125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,2048,0.016721065839131674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,1024,0.016004266341527303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,1536,0.016474666198094685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,65536,0.09385920365651448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,64,0.015173332889874777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,768,0.01586666703224182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,512,0.01548373301823934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,256,0.01514346698919932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,128,0.015230933825174967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,32,0.015106133619944253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,6144,0.02369813323020935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,12288,0.02969813346862793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,7168,0.024794665972391765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,8192,0.025487999121348064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,10240,0.02755733331044515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,5120,0.0206112007300059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,16384,0.03408746719360352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,4096,0.01994346578915914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,3584,0.01956160068511963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,3072,0.01804800033569336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,2048,0.016477866967519125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,2560,0.01699840029080709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,1536,0.01642346680164337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,256,0.015195733308792115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,768,0.015895467003186545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,1024,0.016135467092196147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,65536,0.0813759962717692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,512,0.015352533260981242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,128,0.015003732840220132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,64,0.01513813336690267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,32,0.015037866433461508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,12288,0.027241599559783936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,7168,0.023654399315516154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,10240,0.025566933552424113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,8192,0.02423680027325948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,6144,0.022694400946299233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,16384,0.030780800183614093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,5120,0.02013546625773112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,4096,0.01911679903666178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,2560,0.01651093363761902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,3584,0.017798399925231932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,3072,0.017192532618840538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,2048,0.016597333550453185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,768,0.015787733594576518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,1536,0.01636373301347097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,1024,0.015925332903862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,65536,0.0704639991124471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,256,0.015027200182278952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,512,0.015258666872978211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,64,0.015040000279744467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,128,0.01511146624883016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,32,0.0150709331035614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,8192,0.022836265961329143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,16384,0.02763306697209676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,10240,0.02432639996210734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,12288,0.02539520064989726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,7168,0.022434133291244506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,6144,0.021986132860183714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,5120,0.018866133689880372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,4096,0.017767467101415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,65536,0.060072533289591466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,3584,0.01727679967880249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,1536,0.01625279982884725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,3072,0.017359999815622966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,2048,0.016749866803487144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,2560,0.017115734020868936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,256,0.015278933445612588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,1024,0.01606186628341675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,768,0.015706666310628257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,512,0.015275733669598899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,64,0.015058133006095886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,128,0.01495146652062734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,32,0.015226667126019796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,12288,0.0228767991065979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,16384,0.025049599011739095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,8192,0.02155839999516805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,10240,0.022286933660507203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,7168,0.02121386726697286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,6144,0.01935360034306844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,5120,0.018147200345993042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,65536,0.04852586587270101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,4096,0.01815999945004781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,3584,0.01753386656443278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,1536,0.016316800316174825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,3072,0.017141334215799966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,2560,0.017038933436075845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,2048,0.016692266861597697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,1024,0.015762133399645488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,768,0.01572053333123525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,256,0.015188266833623251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,512,0.015288533767064414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,128,0.014844800035158793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,32,0.015131733814875283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,64,0.01504746675491333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,12288,0.021934932470321654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,10240,0.02123946746190389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,16384,0.023605332771937052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,5120,0.018540799617767334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,7168,0.019887999693552653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,8192,0.020247467358907065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,6144,0.019972266753514607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,65536,0.04257173140843709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,4096,0.018076799313227334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,3584,0.01789120038350423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,3072,0.01725013256072998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,2560,0.017086933056513466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,2048,0.016433067123095193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,1536,0.0162090669075648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,1024,0.015706666310628257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,256,0.015214932958285013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,768,0.0156768004099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,512,0.01548693378766378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,16384,0.022370133797327676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,128,0.015068800250689188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,64,0.01483519971370697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,32,0.014983466267585755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,12288,0.020716800292332967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,10240,0.0201855997244517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,8192,0.019806933403015137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,65536,0.03641706705093384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,7168,0.020172800620396933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,6144,0.019285333156585694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,4096,0.017953066031138103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,5120,0.0186463991800944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,3584,0.01808746655782064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,3072,0.01731946667035421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,2560,0.01691626707712809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,2048,0.01686506668726603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,1024,0.015917866428693136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,256,0.015228799978892007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,1536,0.0160863995552063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,512,0.015497600038846334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,768,0.015590399503707886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,128,0.015054933230082192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,64,0.015146666765213012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,32,0.014985600113868713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,8192,0.019669334093729653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,12288,0.020181334018707274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,16384,0.020140800873438516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,10240,0.020127999782562255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,65536,0.02954453428586324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,5120,0.018573866287867228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,6144,0.019761067628860474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,7168,0.020092799266179403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,4096,0.018092799186706542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,3584,0.01763733426729838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,2048,0.016596266627311708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,3072,0.01726186672846476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,2560,0.017011199394861856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,1536,0.01637333333492279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,1024,0.01590079963207245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,256,0.015093333522478738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,768,0.015660799543062844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,512,0.01515733301639557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,128,0.014883200327555338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,64,0.014985600113868713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,32,0.014972800016403198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,16384,0.020194133122762047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,12288,0.020089600483576456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,65536,0.027110399802525838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,10240,0.020037333170572914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,6144,0.019569067160288493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,5120,0.018631466229756675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,7168,0.01997973322868347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,8192,0.019613866011301676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,2048,0.016619732975959776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,4096,0.018127999703089395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,3584,0.017749333381652833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,2560,0.01703146696090698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,1536,0.016139733791351318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,3072,0.017044266064961754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,1024,0.01569066643714905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,768,0.01546346644560496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,512,0.015379200379053751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,64,0.014987732966740927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,128,0.01490133305390676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,256,0.015251200397809347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,32,0.014737066626548768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,16384,0.02029119928677877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,65536,0.024974934260050454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,10240,0.02002240022023519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,12288,0.019846399625142418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,8192,0.01936533252398173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,7168,0.019987199703852335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,6144,0.019371734062830607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,5120,0.018407466014226277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,3584,0.01763520042101542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,4096,0.017860267559687296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,3072,0.017473065853118898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,2560,0.01687039931615194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,2048,0.01662399967511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,1024,0.015599999825159708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,1536,0.016080000003178916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,768,0.015502933661142984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,256,0.015140266219774882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,512,0.015246933698654175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,128,0.015009066462516785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,32,0.014808533589045205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,64,0.015006933609644571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,16384,0.02009920080502828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,65536,0.024986666440963746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,12288,0.019797333081563315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,8192,0.01943999926249186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,10240,0.019843200842539467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,5120,0.018502400318781535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,7168,0.01994453271230062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,6144,0.019462400674819948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,4096,0.017704532543818156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,3072,0.017246933778127034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,2048,0.016389333208402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,3584,0.017707733313242595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,1536,0.01609173317750295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,2560,0.016810667514801026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,1024,0.015795200069745382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,256,0.015095466375350952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,512,0.015370666980743408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,64,0.014818132917086283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,768,0.01550933321317037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,128,0.015018666783968607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,32,0.014858667055765787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,5120,0.12890666325887043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,6144,0.1509365399678548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,7168,0.17356586456298828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,8192,0.19758399327596027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,10240,0.23979627291361488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,2560,0.07334613005320231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,3584,0.09576106866200765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,4096,0.10628800392150879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,3072,0.08426453272501627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,12288,0.28574399948120116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,256,0.022829866409301756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,512,0.027543467283248902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,2048,0.061476266384124754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,768,0.03403626680374146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,1024,0.03934400081634522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,1536,0.05121279954910278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,128,0.018769067525863648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,64,0.016812799374262492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,32,0.017248000701268515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,16384,0.3791424115498861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,10240,0.07063999970753988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,7168,0.0544543981552124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,8192,0.05978560050328573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,12288,0.08243412971496582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,6144,0.055036799112955725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,16384,0.10295680363972981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,3072,0.03290133277575175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,4096,0.03825066486994426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,5120,0.04384426673253377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,3584,0.035282135009765625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,1536,0.023808000485102336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,2048,0.026809600989023845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,1024,0.021511467297871907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,2560,0.03006933331489563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,768,0.0195413331190745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,512,0.017836799224217735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,256,0.015727999806404113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,128,0.015702399611473083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,64,0.015685333808263143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,32,0.015683199961980185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,8192,0.0495850682258606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,12288,0.06647786696751913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,7168,0.04573226769765218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,10240,0.05830933252970377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,16384,0.083024001121521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,5120,0.03698346614837646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,4096,0.03323306639989217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,6144,0.04136213461558024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,3584,0.03067413369814555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,2560,0.025783467292785644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,3072,0.02821226716041565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,768,0.018606932957967122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,1536,0.021840000152587892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,2048,0.023939200242360435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,1024,0.019771732886632285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,512,0.015718400478363037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,256,0.01569386621316274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,128,0.015435733397801719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,64,0.015631999572118124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,32,0.015634133418401083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,8192,0.04399466514587402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,65536,0.36299839019775393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,16384,0.07190186977386474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,10240,0.051437866687774655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,12288,0.05824640194574991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,5120,0.03347626527150472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,3584,0.02805546720822652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,4096,0.029891200860341388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,7168,0.04080853462219238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,6144,0.0402890682220459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,65536,0.28430401484171547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,3072,0.026052266359329224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,2560,0.024202666680018105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,2048,0.022498132785161336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,1536,0.02074986696243286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,512,0.01571626663208008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,1024,0.018895999590555827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,768,0.01803413430849711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,32,0.015326933066050211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,256,0.015620266397794088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,64,0.015451733271280924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,128,0.015447466572125753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,12288,0.04964906771977742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,7168,0.035454932848612467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,8192,0.038211198647816975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,10240,0.04380906820297241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,16384,0.06003626585006714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,3584,0.02524159948031108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,3072,0.023851732412974037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,65536,0.23821333249409996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,4096,0.026869332790374754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,6144,0.03291200002034505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,5120,0.02987839976946513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,2048,0.02093013326327006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,2560,0.022461867332458495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,1536,0.020530132452646892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,1024,0.01811199982961019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,768,0.015897599856058757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,256,0.015598932902018229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,512,0.015958399573961893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,128,0.015293866395950317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,64,0.015169066190719605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,32,0.015513599912325541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,10240,0.040746665000915526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,16384,0.05500586827596029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,12288,0.04595946470896403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,7168,0.033353598912556966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,8192,0.03560853401819865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,65536,0.19613332748413087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,5120,0.028257066011428834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,4096,0.02539093295733134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,3584,0.02430293361345927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,6144,0.03107306758562724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,3072,0.02285333275794983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,2048,0.020376533269882202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,2560,0.022827732563018798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,1536,0.019261866807937622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,768,0.01583466629187266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,256,0.01544426679611206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,1024,0.01687039931615194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,512,0.015545599659283958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,128,0.015131733814875283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,64,0.015382400155067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,32,0.015185067057609558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,65536,0.17718507448832194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,12288,0.04229653278986613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,7168,0.031542400519053146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,10240,0.03779733180999756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,16384,0.05154666503270468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,8192,0.034169598420461016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,6144,0.029865600665410358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,5120,0.02677653431892395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,4096,0.024600533644358318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,3584,0.023590399821599325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,3072,0.022362667322158813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,2560,0.021065600713094077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,2048,0.019638399283091225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,1024,0.01565439999103546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,1536,0.018595200777053834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,128,0.015140266219774882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,768,0.015780267119407655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,512,0.01569919983545939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,256,0.015502933661142984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,64,0.015161599715550741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,32,0.014999467134475707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,65536,0.1537002722422282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,10240,0.03396693468093872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,12288,0.03752533197402954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,16384,0.043876266479492186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,8192,0.030104533831278486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,7168,0.028266666332880656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,6144,0.026306132475535076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,4096,0.02271359960238139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,5120,0.02451733350753784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,2048,0.019261866807937622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,3072,0.020984532435735066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,3584,0.02163413365681966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,2560,0.019964800278345744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,1536,0.01762346625328064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,768,0.015681067109107973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,1024,0.015799466768900552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,512,0.015621333320935567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,256,0.015144532918930054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,32,0.015210666259129844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,128,0.014910933375358582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,64,0.015309866269429526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,65536,0.13242666721343993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,7168,0.027881600459416706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,10240,0.032441600163777666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,6144,0.02653333346048991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,8192,0.02932479977607727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,16384,0.04131840070088704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,12288,0.03582079807917277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,5120,0.02286826570828756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,3584,0.020826667547225952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,4096,0.021498666206995646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,3072,0.019933867454528808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,2560,0.019381332397460937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,2048,0.018480000893274943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,1024,0.015787733594576518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,1536,0.016523733735084534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,768,0.015794133146603904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,256,0.015310933192571005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,512,0.015358933806419372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,64,0.01524906655152639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,128,0.015131733814875283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,32,0.015422933300336204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,65536,0.10910399754842122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,10240,0.030894933144251506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,8192,0.028128000100453694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,7168,0.02667093276977539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,6144,0.025570134321848553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,16384,0.03841919898986816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,12288,0.033758934338887533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,5120,0.022190932432810465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,3584,0.020156800746917725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,4096,0.0209279994169871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,1536,0.016438399751981102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,2560,0.018962132930755615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,3072,0.01965120037396749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,2048,0.017154133319854735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,512,0.015532799561818442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,1024,0.016035200158754984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,768,0.015595733126004537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,256,0.015109333395957946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,128,0.01511360009511312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,32,0.015119999647140503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,64,0.014894933501879374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,65536,0.09779307047526041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,8192,0.027167999744415285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,10240,0.029513599475224812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,7168,0.025983999172846477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,12288,0.03202986717224121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,6144,0.024756266673405965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,5120,0.021306665738423665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,16384,0.036741332213083906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,4096,0.020295466979344687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,3584,0.019528534015019736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,2048,0.016633599996566772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,2560,0.01802026629447937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,3072,0.019275732835133872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,1536,0.016140799721082053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,512,0.015348266561826071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,1024,0.015991466244061787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,768,0.015493333339691162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,256,0.015280000368754067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,128,0.015024000406265258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,65536,0.08898879686991373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,32,0.01502293348312378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,64,0.01497066617012024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,12288,0.029018666346867877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,7168,0.024445867538452147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,10240,0.02709439992904663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,8192,0.02540480097134908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,6144,0.023466666539510093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,16384,0.033012266953786215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,4096,0.019598933060963948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,5120,0.0206058661142985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,3584,0.01916266679763794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,3072,0.018147200345993042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,1536,0.016300800442695617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,2560,0.016910932461420693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,2048,0.01662613352139791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,1024,0.0159061332543691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,768,0.015770666797955833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,512,0.015573333700497946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,65536,0.07741119861602783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,256,0.015172266960144043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,128,0.015067733327547708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,64,0.01487573285897573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,32,0.01511146624883016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,10240,0.025303467114766436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,12288,0.026570665836334228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,16384,0.029597866535186767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,8192,0.023874133825302124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,7168,0.022962133089701332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,6144,0.02241386572519938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,5120,0.020124799013137816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,4096,0.019112533330917357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,1536,0.016194132963816325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,3584,0.01842026710510254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,2048,0.016616533199946083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,2560,0.016546133160591125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,65536,0.06662400166193644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,3072,0.016989866892496742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,768,0.01570453345775604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,1024,0.015938133001327515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,256,0.015209600329399109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,512,0.01546986699104309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,128,0.014817066987355552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,64,0.015037866433461508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,32,0.014972800016403198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,12288,0.024574933449427287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,16384,0.02690666715304057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,8192,0.02249706586201986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,10240,0.023707733551661173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,7168,0.022188800573349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,6144,0.021602133909861244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,5120,0.018794665733973183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,4096,0.017663999398549398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,3584,0.017633066574732462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,65536,0.055384532610575354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,3072,0.017107200622558594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,1536,0.01618346671263377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,2560,0.01681706706682841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,2048,0.016378666957219443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,128,0.015043200055758158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,768,0.015466666221618653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,1024,0.01593066652615865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,512,0.01532586713631948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,256,0.015226667126019796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,64,0.015007999539375306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,32,0.014961066842079162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,10240,0.02190613349278768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,16384,0.02422719995180766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,12288,0.02237866719563802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,7168,0.020516266425450645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,8192,0.021191465854644775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,5120,0.01848213275273641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,6144,0.019451733430226645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,65536,0.04454186757405599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,4096,0.01798293391863505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,2048,0.016410666704177856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,3072,0.017383466164271034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,3584,0.017525333166122436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,2560,0.016960000991821288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,1536,0.01601066688696543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,512,0.01532906691233317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,1024,0.015899733702341715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,768,0.01537493367989858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,256,0.015083733201026916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,128,0.015245866775512696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,64,0.015160533785820007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,32,0.014958932995796204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,16384,0.023256532351175942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,12288,0.02155733307202657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,10240,0.020827732483545938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,8192,0.01964906652768453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,7168,0.019639466206232706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,65536,0.038328532377878824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,5120,0.018598399559656777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,6144,0.019563732544581096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,4096,0.017594667275746663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,3584,0.017425066232681273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,1536,0.016139733791351318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,2048,0.01651413341363271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,3072,0.01731626590092977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,2560,0.016822399695714314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,768,0.01537493367989858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,1024,0.01569919983545939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,512,0.015461333592732749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,256,0.015215999881426492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,128,0.014916266997655234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,64,0.015154133240381876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,32,0.015150933464368185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,16384,0.021612799167633055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,12288,0.020076799392700195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,10240,0.01991466681162516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,8192,0.019367466370264687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,65536,0.03237653374671936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,6144,0.01957546671231588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,7168,0.02012373407681783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,5120,0.018636800845464072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,3072,0.01737706661224365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,3584,0.017540266116460167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,4096,0.01789120038350423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,2560,0.016772266228993735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,2048,0.016693333784739174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,768,0.015431466698646545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,1536,0.01616106629371643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,1024,0.015719466408093772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,512,0.015290666619936624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,256,0.015229866902033488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,128,0.01476800044377645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,32,0.014872533082962037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,64,0.015173332889874777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,12288,0.019613866011301676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,16384,0.01979413429896037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,10240,0.019766400257746376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,65536,0.026730666557947796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,8192,0.01921280026435852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,7168,0.019612799088160195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,5120,0.018655999501546224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,6144,0.01954986651738485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,3584,0.01758079926172892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,4096,0.01770346760749817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,3072,0.01734613378842672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,1536,0.016107733050982155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,2560,0.016663466890652977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,2048,0.01638826628526052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,1024,0.015778133273124696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,768,0.015432533621788026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,128,0.014985600113868713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,512,0.015135999520619711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,256,0.015226667126019796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,32,0.014828800161679586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,64,0.014745600024859109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,12288,0.019413334131240845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,16384,0.01990293264389038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,65536,0.02306879957516988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,7168,0.019527467091878255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,6144,0.019107200702031455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,10240,0.01960106690724691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,8192,0.019248000780741372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,5120,0.018334933121999106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,4096,0.017783466974894205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,3072,0.01716586748758952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,3584,0.017429333925247193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,2560,0.016612266500790916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,1536,0.016057599584261575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,2048,0.016272000471750894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,1024,0.01572480003039042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,512,0.01514346698919932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,768,0.01544319987297058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,128,0.014739200472831726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,256,0.015149866541226705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,16384,0.019589332739512126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,64,0.015010133385658264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,32,0.014854400356610616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,65536,0.02178666591644287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,12288,0.019236266613006592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,10240,0.01925546725591024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,8192,0.019297067324320474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,7168,0.01967573364575704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,4096,0.01770133376121521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,6144,0.019403733809789023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,3584,0.017333332697550455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,5120,0.018232532342274985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,3072,0.016835200786590575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,2048,0.016355199615160625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,2560,0.016717867056528727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,1536,0.015994667013486227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,768,0.015384533007939658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,1024,0.015842133760452272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,512,0.015141333142916361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,128,0.014914133151372275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,256,0.014934399724006652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,32,0.014979199568430582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,64,0.015081600348154704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,65536,0.022285866737365722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,10240,0.019509333372116088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,16384,0.019672532876332603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,12288,0.01917440096537272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,8192,0.01891626715660095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,7168,0.0194432000319163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,6144,0.019412267208099365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,5120,0.01841813325881958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,4096,0.017733333508173625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,2048,0.016382933656374613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,3584,0.017448532581329345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,2560,0.01644373337427775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,3072,0.016849066813786825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,1536,0.0160671999057134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,768,0.015421866377194723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,1024,0.015687466661135355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,512,0.015245866775512696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,128,0.014737066626548768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,256,0.015277866522471109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,64,0.014845866958300272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,32,0.015044266978899637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,5120,0.12810453573862712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,6144,0.15071573257446289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,7168,0.17287893295288087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,8192,0.19691413243611652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,10240,0.2391424020131429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,4096,0.10443092981974285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,3584,0.09506133397420248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,3072,0.08390293121337891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,12288,0.284550412495931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,2560,0.07227306365966797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,2048,0.06365120013554891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,1536,0.05020159880320231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,128,0.018574933211008705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,256,0.021303466955820718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,512,0.029589333136876422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,768,0.03360213438669841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,1024,0.038729600111643475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,32,0.01699413259824117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,64,0.016822399695714314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,16384,0.37742506663004555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,8192,0.059486933549245204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,7168,0.05423253377278646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,6144,0.05463466644287109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,10240,0.07078933715820312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,12288,0.08179199695587158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,16384,0.10293760299682617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,3584,0.03575466473897298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,4096,0.037894399960835774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,3072,0.03265066742897034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,2560,0.030444800853729248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,5120,0.043322666486104326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,1536,0.023616000016530355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,2048,0.026610134045283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,128,0.015402666727701821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,768,0.019394133488337198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,1024,0.020997333526611327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,256,0.015895467003186545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,512,0.018131200472513834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,64,0.01558080017566681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,32,0.015345066785812378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,7168,0.044913065433502194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,8192,0.0489898681640625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,10240,0.057418668270111085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,12288,0.06625813245773315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,16384,0.0826527992884318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,6144,0.040934399763743086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,5120,0.03681813478469849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,3584,0.030393600463867188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,4096,0.032986666758855185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,3072,0.028190932671229046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,1536,0.021894399325052896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,2560,0.025733333826065064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,2048,0.024011733134587605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,1024,0.019859200716018675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,512,0.015711999932924905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,768,0.01862186590830485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,256,0.0156768004099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,128,0.015455999970436096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,64,0.015545599659283958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,32,0.01548906664053599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,8192,0.04374293486277263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,12288,0.057922132809956874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,10240,0.05098986625671387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,65536,0.35822718938191733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,16384,0.07181653181711832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,7168,0.04025813341140747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,3584,0.0278656005859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,6144,0.03684053421020508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,5120,0.03337706724802653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,4096,0.03184319933255513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,2560,0.024098134040832518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,3072,0.025865600506464644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,2048,0.02220906615257263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,1024,0.018822399775187175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,1536,0.02071466644605001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,768,0.01754666765530904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,65536,0.2840330759684245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,512,0.015797332922617594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,256,0.0153546671072642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,128,0.015059199929237366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,32,0.015211733182271323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,64,0.015385599931081137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,8192,0.03931946754455566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,7168,0.035104000568389894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,10240,0.0432586669921875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,12288,0.049097601572672525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,16384,0.0591317335764567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,6144,0.03245439926783244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,3584,0.02511253356933594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,3072,0.02369920015335083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,4096,0.026449066400527955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,65536,0.2370698610941569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,5120,0.029632000128428142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,1536,0.019556266069412232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,2560,0.022457599639892578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,2048,0.0209824005762736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,1024,0.017862399419148765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,768,0.015587199727694193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,512,0.015833600362141927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,256,0.015507200360298156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,64,0.015314132968584696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,128,0.015092266599337259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,32,0.01513920029004415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,8192,0.0352565328280131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,10240,0.043137065569559735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,12288,0.048648532231648764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,7168,0.032994133234024045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,16384,0.05810346603393555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,4096,0.025175466140111284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,65536,0.19423786799112958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,6144,0.030526934067408244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,3584,0.024216532707214355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,5120,0.028123732407887774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,3072,0.023004800081253052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,2560,0.021580799420674642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,1536,0.019036799669265747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,2048,0.02140053311983744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,1024,0.01618346671263377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,768,0.015635200341542563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,512,0.015635200341542563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,256,0.015378133455912272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,128,0.015090133746465048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,64,0.0151146670182546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,32,0.014972800016403198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,10240,0.03714453379313151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,12288,0.04225279887517293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,7168,0.031267199913660684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,8192,0.033693865935007734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,65536,0.17670826911926268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,16384,0.050910933812459314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,5120,0.026140799125035603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,6144,0.02858346700668335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,4096,0.024452267090479533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,3584,0.023398399353027344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,3072,0.02332373261451721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,2560,0.021130667130152384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,2048,0.019592533508936562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,1536,0.018614399433135986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,1024,0.01551466683546702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,768,0.01562346617380778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,128,0.01514346698919932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,512,0.01541866660118103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,256,0.015475199619928996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,64,0.015089066823323569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,32,0.014947199821472168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,8192,0.03004266619682312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,65536,0.15171200434366863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,12288,0.0373802661895752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,7168,0.027872000137964887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,10240,0.0335477352142334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,16384,0.04370453357696533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,6144,0.026209066311518352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,5120,0.02444373369216919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,3584,0.021733333667119346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,4096,0.022703999280929567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,2048,0.019012266397476198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,3072,0.020933334032694498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,2560,0.020105600357055664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,1536,0.017473065853118898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,512,0.015449600418408713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,1024,0.01569706698258718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,768,0.015552000204722086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,128,0.015209600329399109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,256,0.015461333592732749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,32,0.01507306694984436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,64,0.015076266725858054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,7168,0.025405865907669068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,8192,0.027292799949645997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,65536,0.12593279679616293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,12288,0.03512959877649943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,10240,0.03256426652272542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,16384,0.04061013460159302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,5120,0.022376533349355063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,6144,0.02390506664911906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,4096,0.021217066049575805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,3584,0.020426666736602782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,3072,0.01988906661669413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,2560,0.019139200448989868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,2048,0.01745706597963969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,1536,0.015875200430552162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,768,0.015667200088500977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,128,0.014863999684651694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,1024,0.01569066643714905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,512,0.015593600273132325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,256,0.015398400028546652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,64,0.015084800124168397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,65536,0.1067850669225057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,32,0.015079466501871744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,8192,0.02563626567522685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,6144,0.023126399517059325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,7168,0.024266666173934935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,10240,0.0307093342145284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,12288,0.03316160043080647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,16384,0.03792426586151123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,4096,0.020566399892171225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,5120,0.021917865673700968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,3584,0.019853866100311278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,1536,0.016049066185951234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,2560,0.01877546707789103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,3072,0.019449599583943687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,2048,0.01739733417828878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,1024,0.015825066963831583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,512,0.015525333086649575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,256,0.015361066659291586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,768,0.015370666980743408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,128,0.015239466230074564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,65536,0.09644052982330323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,64,0.01506239970525106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,32,0.015118933717409768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,8192,0.027030400435129803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,10240,0.02879146734873454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,7168,0.025813333193461102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,6144,0.024719999233881632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,12288,0.030701865752538044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,5120,0.021320533752441407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,16384,0.03529599905014038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,2048,0.016394666830698647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,3072,0.019042134284973145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,4096,0.020500266551971437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,3584,0.01946453253428141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,2560,0.01816320021947225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,1536,0.016105600198109946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,1024,0.01585813363393148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,768,0.015668267011642457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,512,0.015592533349990844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,128,0.014846932888031007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,256,0.015175466736157736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,65536,0.08405973116556803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,32,0.015032533804575601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,64,0.015254400173823037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,7168,0.024238934119542442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,12288,0.028585600852966308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,8192,0.025114667415618897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,10240,0.02672533392906189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,6144,0.023502933979034423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,16384,0.03250773350397746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,5120,0.020797866582870483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,3584,0.018984532356262206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,4096,0.019385600090026857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,3072,0.01804479956626892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,1536,0.016104533274968465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,2048,0.0166293332974116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,2560,0.016598400473594666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,1024,0.01572053333123525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,768,0.015454933047294617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,65536,0.07488319873809815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,256,0.015093333522478738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,512,0.015320533514022827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,128,0.01524906655152639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,64,0.015219199657440185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,32,0.015017599860827128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,12288,0.026096000274022417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,16384,0.029158399502436323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,10240,0.024937599897384644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,8192,0.023781333367029825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,7168,0.02304533322652181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,6144,0.022265599171320597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,5120,0.019720532496770225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,4096,0.018942934274673463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,3072,0.017065600554148356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,1536,0.016049066185951234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,3584,0.01718719998995463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,2048,0.016552533706029257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,2560,0.01681813398996989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,65536,0.06454293330510458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,512,0.015301332871119181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,1024,0.015738667050997416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,128,0.0150026669104894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,768,0.015655466914176942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,256,0.015129599968592325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,64,0.015240533153216043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,32,0.014857600132624308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,12288,0.02443839907646179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,16384,0.026552534103393553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,10240,0.023643734057744344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,6144,0.021282132466634116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,8192,0.022453333934148154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,7168,0.02203413248062134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,5120,0.019817600647608437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,4096,0.018023467063903807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,3584,0.0176746666431427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,65536,0.052799999713897705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,3072,0.017288533846537273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,1536,0.016160000363985697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,2560,0.016612266500790916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,2048,0.016603733102480568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,1024,0.015844266613324484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,128,0.015014400084813436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,512,0.015318399667739869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,768,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,256,0.01493333379427592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,64,0.01497706671555837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,32,0.015172266960144043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,16384,0.023971199989318848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,12288,0.02222933371861776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,8192,0.020797866582870483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,10240,0.021831466754277547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,7168,0.02076906760533651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,6144,0.019256534179051717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,5120,0.018487467368443807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,65536,0.04226239919662476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,4096,0.017910399039586387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,3584,0.017373865842819212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,3072,0.017218132813771568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,1536,0.01611733337243398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,2048,0.016536532839139303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,2560,0.01669013301531474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,1024,0.015809067090352378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,768,0.015526400009791056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,512,0.015158399939537048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,256,0.015318399667739869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,128,0.015006933609644571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,64,0.01520639955997467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,32,0.014958932995796204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,16384,0.022554665803909302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,8192,0.01996906598409017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,10240,0.020706133047739664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,12288,0.021115734179814657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,65536,0.036958932876586914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,5120,0.018465065956115724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,7168,0.01936960021654765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,6144,0.019167999426523842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,4096,0.017821866273880004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,3584,0.01757226586341858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,3072,0.017030400037765504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,2560,0.01681706706682841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,768,0.015562666455904641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,2048,0.01644373337427775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,1536,0.016207999984423318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,1024,0.015666133165359496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,512,0.015409066279729208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,256,0.014910933375358582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,128,0.014963199694951376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,64,0.015033599734306336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,32,0.014937600493431092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,16384,0.02113706668217977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,10240,0.01944533387819926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,12288,0.019262933731079103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,7168,0.019605332612991334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,8192,0.019287467002868652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,6144,0.019223467508951823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,65536,0.03081386685371399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,3072,0.017128533124923705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,4096,0.017633066574732462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,5120,0.018496000766754152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,2048,0.01634666621685028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,3584,0.017629865805308023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,2560,0.016875733931859337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,1536,0.016217600305875143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,1024,0.015727999806404113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,256,0.014922666549682616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,768,0.015412267049153647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,128,0.015221333503723145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,64,0.014990933736165366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,512,0.015177599589029946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,32,0.01516480048497518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,16384,0.019330133994420372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,65536,0.024727465709050496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,12288,0.01899306575457255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,8192,0.019211733341217042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,10240,0.01990506649017334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,7168,0.019478400548299156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,5120,0.01840426723162333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,6144,0.019321600596110024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,4096,0.017825067043304443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,3584,0.01737173398335775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,3072,0.01707306702931722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,1536,0.01602026621500651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,2560,0.016858667135238647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,2048,0.01639573375384013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,768,0.015526400009791056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,1024,0.015785599748293556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,512,0.015141333142916361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,256,0.014932266871134438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,128,0.014819199840227762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,65536,0.022766933838526408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,64,0.015075199802716575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,32,0.014841600259145101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,16384,0.01951040029525757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,12288,0.018997333447138467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,10240,0.01944213310877482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,8192,0.018969599405924478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,7168,0.019426133235295615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,6144,0.019233065843582153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,5120,0.0181877334912618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,2560,0.01660586694876353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,4096,0.017730132738749186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,3072,0.017092265685399375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,3584,0.017208532492319743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,2048,0.016369066635767617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,1024,0.01544319987297058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,1536,0.016056533654530844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,768,0.015431466698646545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,64,0.014940800269444785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,256,0.015110400319099427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,128,0.01474453310171763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,512,0.015330132842063905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,32,0.014847999811172486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,12288,0.01911146640777588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,16384,0.019293866554896035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,65536,0.020653865734736123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,10240,0.019462400674819948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,8192,0.018872533241907755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,7168,0.019641600052515664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,5120,0.01809599995613098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,6144,0.018925867478052773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,4096,0.017571200927098594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,3072,0.016978132724761962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,3584,0.01735146641731262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,2560,0.01649493376413981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,1536,0.01586026648680369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,2048,0.016425599654515587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,768,0.015406933426856995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,1024,0.015428266922632852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,512,0.015296000242233276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,128,0.01474346617857615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,256,0.01520746648311615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,64,0.014942933122316995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,32,0.015090133746465048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,65536,0.02072319984436035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,16384,0.01913706660270691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,12288,0.018899200359980266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,7168,0.019334399700164796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,10240,0.019139200448989868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,8192,0.019041067361831664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,6144,0.019036799669265747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,3072,0.01685653328895569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,5120,0.017988266547520955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,3584,0.017244799931844076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,4096,0.01735360026359558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,2560,0.016364799936612447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,2048,0.016335999965667723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,1024,0.015499732891718545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,768,0.015202132860819497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,1536,0.01601066688696543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,512,0.015284267067909241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,128,0.014724266529083253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,256,0.014906666676203408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,64,0.014944000045458474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,32,0.01495573321978251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,5120,0.1274293343226115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,6144,0.14971092542012532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,7168,0.17284693717956542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,8192,0.19650774002075194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,4096,0.1046229362487793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,10240,0.2380298614501953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,3584,0.09446826775868734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,12288,0.2839861234029134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,3072,0.08327893416086832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,2560,0.07194666862487793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,1536,0.05373013416926066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,2048,0.06039040088653565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,1024,0.0385696013768514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,256,0.021261866887410483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,64,0.019614932934443156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,512,0.027058132489522296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,128,0.020594133933385213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,768,0.036432000001271565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,32,0.019747199614842732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,16384,0.37645759582519533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,7168,0.05398826599121094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,8192,0.059324800968170166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,6144,0.05437759955724081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,10240,0.06982186635335287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,12288,0.08171199957529704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,16384,0.10169386863708496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,3584,0.03498986562093099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,3072,0.032289065917332965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,4096,0.03770346641540527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,5120,0.0432426651318868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,2560,0.029531733194986982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,1536,0.02373973329861959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,1024,0.02076906760533651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,2048,0.026676267385482788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,768,0.01962239940961202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,512,0.017876267433166504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,128,0.015340800086657206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,256,0.015796266992886863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,64,0.015533866484959922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,32,0.015471999843915304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,7168,0.04468799829483032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,8192,0.04973866542180379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,12288,0.06549013455708821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,10240,0.056518399715423585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,16384,0.08288959662119547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,6144,0.04066239992777507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,4096,0.032384000221888226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,5120,0.03641066551208496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,3584,0.030053333441416426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,3072,0.028218666712443035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,2048,0.02371413310368856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,2560,0.025781333446502686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,1536,0.02199466625849406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,1024,0.019706666469573975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,512,0.01591146687666575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,256,0.015828266739845276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,768,0.0185589333375295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,128,0.01551466683546702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,32,0.015442132949829102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,64,0.015442132949829102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,10240,0.05045973459879557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,12288,0.05765866835912069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,8192,0.042771200338999435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,65536,0.35744533538818357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,16384,0.0708725372950236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,7168,0.04015680154164632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,6144,0.03670080105463664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,5120,0.03336426814397176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,3584,0.027863466739654542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,4096,0.031854933500289916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,3072,0.02582826614379883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,768,0.016597333550453185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,2560,0.02409279942512512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,65536,0.2799541473388672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,1536,0.020588799317677816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,1024,0.01883626580238342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,2048,0.022201599677403767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,512,0.015778133273124696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,256,0.015498666961987814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,64,0.015432533621788026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,32,0.015069866180419922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,128,0.015129599968592325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,7168,0.034997332096099856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,16384,0.058804265658060705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,8192,0.03858240048090617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,10240,0.04307093222935994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,12288,0.04887679815292358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,65536,0.23629439671834312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,3584,0.025214932362238568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,3072,0.023715200026830037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,5120,0.02944213350613912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,4096,0.026338134209314985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,6144,0.03237973252932231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,2560,0.02404159903526306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,2048,0.020936532815297445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,1536,0.0195360004901886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,1024,0.018040533860524496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,768,0.01558080017566681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,512,0.01601066688696543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,256,0.015525333086649575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,32,0.015219199657440185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,128,0.015387733777364096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,64,0.015202132860819497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,10240,0.04276693264643351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,7168,0.03277013301849365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,8192,0.035290666421254474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,12288,0.04772160053253174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,16384,0.0574346661567688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,65536,0.19293120702107747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,3584,0.024124799172083537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,4096,0.025370667378107708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,5120,0.02778559923171997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,6144,0.03057813247044881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,2048,0.02029013236363729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,3072,0.02299626668294271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,1536,0.019088000059127808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,2560,0.022817067305246987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,1024,0.01690346598625183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,512,0.01581760048866272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,768,0.01565120021502177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,256,0.015528532862663268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,128,0.015306666493415833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,64,0.015082666277885437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,32,0.0151936004559199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,16384,0.05019199848175049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,10240,0.037963732083638506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,8192,0.03362773259480794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,7168,0.031196800867716472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,12288,0.042414931456247966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,65536,0.1743029276529948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,6144,0.02895146608352661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,3584,0.023324799537658692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,5120,0.026841600735982258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,4096,0.024465066194534302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,3072,0.02219520012537638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,2560,0.02190613349278768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,2048,0.019783467054367065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,1536,0.018631466229756675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,128,0.015057067076365152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,1024,0.015809067090352378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,768,0.01574613352616628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,512,0.015777066349983215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,256,0.015156267086664834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,32,0.015094400445620219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,64,0.015171200037002563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,65536,0.15004587173461914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,8192,0.029676800966262816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,10240,0.03308266599973043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,7168,0.02808213432629903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,12288,0.03715733289718628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,16384,0.04314133326212565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,5120,0.02413653333981832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,6144,0.02608319918314616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,4096,0.022345600525538127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,3584,0.021706666549046835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,3072,0.02092906634012858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,2560,0.019938133160273232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,2048,0.019022933642069497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,1536,0.017417599757512413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,1024,0.015924266974131265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,768,0.01583466629187266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,128,0.014935466647148132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,512,0.015707733233769734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,256,0.015217066804567973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,64,0.015096533298492431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,32,0.014922666549682616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,65536,0.12466560204823811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,8192,0.028926932811737062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,7168,0.025164800882339477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,12288,0.034991999467213944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,16384,0.04028373161951701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,10240,0.03222399950027466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,6144,0.024089600642522177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,5120,0.022605866193771362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,4096,0.02099626660346985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,3072,0.01988053321838379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,3584,0.020610133806864418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,2560,0.019228800137837728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,1536,0.016126933693885803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,2048,0.018225065867106118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,768,0.015576533476511636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,1024,0.01566506624221802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,512,0.015692800283432007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,256,0.015153066317240397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,128,0.0150709331035614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,32,0.015059199929237366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,64,0.015152000387509666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,65536,0.10549226601918538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,8192,0.025306665897369386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,6144,0.02302079995473226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,7168,0.024284799893697105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,10240,0.03028693397839864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,5120,0.021840000152587892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,16384,0.0375434676806132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,12288,0.03272853295008342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,4096,0.020518400271733604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,3584,0.020038400093714395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,2560,0.018869332472483315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,3072,0.01945919990539551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,2048,0.01742186745007833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,1536,0.015988266468048094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,512,0.015421866377194723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,1024,0.015777066349983215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,768,0.015796266992886863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,256,0.01520639955997467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,65536,0.09601600170135498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,32,0.015020799636840821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,128,0.015244799852371215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,64,0.01502293348312378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,8192,0.026824533939361572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,10240,0.02842346628506978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,7168,0.025913600126902265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,6144,0.024502400557200113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,12288,0.030473599831263225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,16384,0.03509013255437215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,5120,0.020994132757186888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,4096,0.019883733987808228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,1536,0.015636266271273295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,3584,0.019573332866032918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,3072,0.018922666708628334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,2560,0.01808639963467916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,2048,0.016551466782887776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,768,0.015424000223477683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,1024,0.015525333086649575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,65536,0.08507733345031739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,512,0.015656532843907674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,128,0.015086932977040609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,256,0.0151637335618337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,64,0.015186132987340293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,32,0.014932266871134438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,6144,0.023321600755055745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,12288,0.028673066695531206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,10240,0.02677759925524394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,8192,0.02486613392829895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,7168,0.024276266495386757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,16384,0.032304000854492185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,5120,0.020678400993347168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,4096,0.019460266828536986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,3584,0.01903146704037984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,3072,0.01700053413709005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,2048,0.01646719972292582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,2560,0.016727467377980552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,1536,0.016125866770744325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,65536,0.07338666915893555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,64,0.014988799889882406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,768,0.015577600399653117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,1024,0.015929599603017174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,256,0.015190399686495461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,512,0.01537493367989858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,128,0.015268266201019287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,32,0.014917332927385965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,10240,0.02510506709416707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,12288,0.0261354664961497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,16384,0.028899200757344562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,8192,0.023579732577006022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,7168,0.02304746707280477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,6144,0.022170666853586832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,3584,0.017488000790278117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,5120,0.01992853283882141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,4096,0.018979199727376304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,2560,0.01683626572291056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,3072,0.016752000649770102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,1536,0.016064000129699708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,65536,0.06321599880854288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,2048,0.016167466839154564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,1024,0.01583253343900045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,768,0.015713066856066386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,128,0.015043200055758158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,512,0.015560533603032431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,256,0.01511573294798533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,64,0.014987732966740927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,32,0.01483519971370697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,12288,0.024171733856201173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,16384,0.026578134298324584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,7168,0.02176213264465332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,10240,0.023891200621922813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,6144,0.021127466360727945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,8192,0.02234026590983073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,3584,0.01746986707051595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,5120,0.019410133361816406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,4096,0.017746132612228394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,65536,0.051870934168497716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,2560,0.016614400347073875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,3072,0.017169066270192466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,2048,0.016360533237457276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,1536,0.016141866644223533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,1024,0.015822933117548624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,512,0.015400532881418863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,768,0.015717333555221556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,256,0.014932266871134438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,128,0.015255467096964518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,32,0.014968533317248026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,64,0.014947199821472168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,7168,0.019671465953191122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,16384,0.02372586727142334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,10240,0.021602133909861244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,12288,0.022102399667104086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,8192,0.020949333906173706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,6144,0.019170133272806804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,5120,0.0183242658774058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,65536,0.041015466054280594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,4096,0.017698132991790773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,3584,0.017203199863433837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,3072,0.017167999347050985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,2560,0.016847999890645345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,1536,0.015964800119400026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,2048,0.01628373364607493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,1024,0.015709867080052696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,768,0.01534293293952942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,64,0.015264000495274863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,512,0.01514240006605784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,256,0.015287466843922935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,128,0.015549866358439126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,32,0.015338666240374246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,12288,0.021215999126434328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,16384,0.022729599475860597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,10240,0.020331732432047524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,5120,0.018357332547505698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,8192,0.0191210667292277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,7168,0.019372800985972084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,65536,0.035734399159749346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,6144,0.019207467635472618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,4096,0.01765759984652201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,3584,0.017314134041468303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,3072,0.017026132345199584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,1536,0.0160863995552063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,2560,0.0168778657913208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,2048,0.016568533579508462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,1024,0.01562773287296295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,256,0.015130666891733804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,768,0.01588586668173472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,512,0.01530880033969879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,128,0.015123200416564942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,16384,0.02129813234011332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,32,0.015269333124160766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,64,0.01527253290017446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,12288,0.01907306710879008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,10240,0.019237333536148073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,7168,0.0194815993309021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,65536,0.02977386713027954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,8192,0.01902186671892802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,6144,0.019370667139689126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,5120,0.018692266941070557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,4096,0.01786880095799764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,3584,0.017321600516637167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,3072,0.01705706715583801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,2048,0.01649066706498464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,2560,0.01681493322054545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,768,0.015612799922625223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,256,0.015118933717409768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,1024,0.01598186691602071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,1536,0.016238933801651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,512,0.015238400300343832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,128,0.015076266725858054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,64,0.014973866939544677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,32,0.01511573294798533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,65536,0.02468799948692322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,10240,0.019191465775171914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,16384,0.019337600469589232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,12288,0.019211733341217042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,8192,0.01903466582298279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,5120,0.01834133267402649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,7168,0.019478400548299156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,6144,0.019205333789189656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,4096,0.01758400003115336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,3584,0.017541333039601644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,3072,0.01723626653353373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,2560,0.016773333152135216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,2048,0.016323199868202208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,1024,0.015754666924476624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,1536,0.015953066945075988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,768,0.015334399541219077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,64,0.014853333433469137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,512,0.015532799561818442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,128,0.015026133259137472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,256,0.015217066804567973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,32,0.015059199929237366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,65536,0.021810134251912437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,12288,0.018923733631769815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,16384,0.019090133905410766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,8192,0.018908800681432088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,10240,0.019355734189351402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,7168,0.019326933224995933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,6144,0.019063466787338258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,4096,0.017438934246699015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,3584,0.017163733641306557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,3072,0.016915200153986613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,5120,0.018051199118296304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,2048,0.016323199868202208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,2560,0.016679465770721436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,1536,0.01606826682885488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,1024,0.01565333306789398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,512,0.015337600310643514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,768,0.015280000368754067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,256,0.0150218665599823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,32,0.014904533823331198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,128,0.015126400192578635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,64,0.015140266219774882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,16384,0.018988800048828126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,65536,0.020181334018707274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,12288,0.018773333231608073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,10240,0.019101866086324058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,8192,0.01872533361117045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,5120,0.01796906590461731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,4096,0.017574399709701538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,7168,0.01919893423716227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,6144,0.019118932882944743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,3584,0.017056000232696534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,2560,0.016403200229008992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,2048,0.016090666254361473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,3072,0.016723199685414632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,1536,0.015786666671435037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,1024,0.015691733360290526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,768,0.015465600291887918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,512,0.015057067076365152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,256,0.015026133259137472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,128,0.014958932995796204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,32,0.01493119994799296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,64,0.014873600006103516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,65536,0.020181334018707274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,16384,0.019339734315872194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,12288,0.018708266814549766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,7168,0.019220266739527384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,10240,0.01919680039087931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,8192,0.0187285323937734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,6144,0.018795732657114664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,5120,0.018145066499710084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,3072,0.016808533668518068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,3584,0.017026132345199584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,2048,0.016130133469899496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,4096,0.017598932981491087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,2560,0.016531200210253397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,768,0.015355733036994935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,256,0.015031466881434122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,1536,0.015808000167210897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,512,0.015032533804575601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,64,0.014968533317248026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,1024,0.015687466661135355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,128,0.014921599626541137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,32,0.014859732985496522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,5120,0.12687679926554363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,6144,0.14953920046488445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,7168,0.1715221405029297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,8192,0.19509973526000976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,4096,0.10382826328277588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,10240,0.23737492561340331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,3584,0.09381226698557535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,3072,0.08300800323486328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,2560,0.0717578649520874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,12288,0.28454720179239906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,2048,0.05996906757354736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,1024,0.03824746608734131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,768,0.03307200074195862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,1536,0.04986879825592041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,512,0.02712106704711914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,128,0.020477867126464842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,256,0.022578134139378866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,64,0.0196234663327535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,32,0.019598933060963948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,16384,0.37538986206054686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,7168,0.05379946629206339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,12288,0.08114666938781738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,8192,0.05914346774419149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,10240,0.07021120389302572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,6144,0.05462186733881632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,16384,0.10190400282541912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,4096,0.037411201000213626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,3072,0.032229334115982056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,3584,0.03504853248596192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,2560,0.029322665929794312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,5120,0.043109333515167235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,2048,0.026446932554244997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,512,0.017844265699386595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,1536,0.02360960046450297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,768,0.01964906652768453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,1024,0.020898133516311646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,256,0.01585813363393148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,128,0.015496533115704855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,64,0.015492266416549683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,32,0.015457066893577575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,7168,0.04453866481781006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,8192,0.0486624002456665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,10240,0.05650986830393473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,12288,0.06675519943237304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,16384,0.08158079783121744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,5120,0.03630719979604085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,6144,0.040396801630655926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,4096,0.032035199801127116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,3584,0.030055467287699384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,2048,0.023642667134602866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,2560,0.02569920023282369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,3072,0.027827199300130206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,1536,0.021692800521850585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,1024,0.019706666469573975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,512,0.01647040049235026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,768,0.01861226757367452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,128,0.01548799971739451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,256,0.01567146678765615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,64,0.015572266777356467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,32,0.015500799814860026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,8192,0.04325439929962158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,10240,0.050581332047780356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,12288,0.05684586763381958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,65536,0.35713494618733727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,16384,0.07061013380686441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,7168,0.03983893394470215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,3584,0.02778666615486145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,4096,0.03164373238881429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,6144,0.03675306638081868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,3072,0.025740800301233928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,5120,0.032696533203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,65536,0.279146671295166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,1024,0.01882879932721456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,2560,0.024119466543197632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,2048,0.022194133202234904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,1536,0.02058666745821635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,768,0.01685653328895569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,256,0.015589333573977151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,512,0.015401599804560342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,128,0.015296000242233276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,64,0.015052800377209982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,32,0.015324800213177999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,8192,0.038371201356252035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,7168,0.03481920162836711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,16384,0.05862400134404501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,10240,0.04296853144963582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,12288,0.048715734481811525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,65536,0.23235839207967124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,3584,0.025174399216969807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,4096,0.026391466458638508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,3072,0.023639466365178427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,6144,0.03254080017407735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,5120,0.02950719992319743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,2560,0.023824000358581544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,2048,0.020963199933369956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,1536,0.01953173279762268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,1024,0.01797973314921061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,768,0.015793066223462424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,512,0.015946666399637856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,64,0.015265066425005594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,256,0.01548479994138082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,128,0.015309866269429526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,32,0.015245866775512696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,10240,0.04243306716283162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,8192,0.034808532396952314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,12288,0.048121599356333415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,16384,0.05705386797587077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,7168,0.03301440080006917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,65536,0.19186879793802897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,4096,0.025388799111048382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,6144,0.03020053307215373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,5120,0.02800319989522298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,3584,0.024133332570393882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,3072,0.022909865776697794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,2048,0.020398932695388793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,1536,0.01975359916687012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,2560,0.02283733288447062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,1024,0.016732800006866454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,768,0.01620693306128184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,512,0.015633066495259605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,128,0.015057067076365152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,256,0.015446399648984274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,64,0.01511360009511312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,32,0.015067733327547708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,10240,0.037384533882141115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,12288,0.04164053201675415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,7168,0.030922667185465498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,8192,0.033600000540415446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,16384,0.05133440097173055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,65536,0.17364586194356282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,5120,0.026547199487686156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,4096,0.02439893285433451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,6144,0.0288480003674825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,2048,0.019732266664505005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,3584,0.023336533705393472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,3072,0.02217493255933126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,2560,0.021971199909845987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,1536,0.018410666783650716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,768,0.015847466389338174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,1024,0.015796266992886863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,512,0.015544533729553223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,256,0.01551040013631185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,64,0.015155200163523355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,128,0.015003732840220132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,32,0.015086932977040609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,7168,0.027910399436950683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,65536,0.14947519302368165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,8192,0.02945493261019389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,10240,0.033242666721343996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,12288,0.03698133230209351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,6144,0.02592639923095703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,16384,0.042898134390513105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,2560,0.02014933427174886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,5120,0.024312533934911094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,4096,0.02228906750679016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,3584,0.0218560000260671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,3072,0.021025067567825316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,2048,0.01891626715660095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,1536,0.01741973360379537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,1024,0.01599253316720327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,512,0.015542399883270264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,768,0.015528532862663268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,64,0.015094400445620219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,256,0.015410133202870688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,128,0.015066666404406228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,32,0.015184000134468079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,65536,0.12672746976216634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,12288,0.03456319967905681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,7168,0.02524906595547994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,10240,0.03217066725095113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,16384,0.03988693157831828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,8192,0.029158399502436323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,6144,0.024038400252660116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,5120,0.022475733359654745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,4096,0.02127359906832377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,3072,0.019755733013153077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,3584,0.020538665850957236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,2048,0.018210132916768394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,2560,0.019105066855748497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,1536,0.01586133340994517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,1024,0.015998933712641397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,256,0.015238400300343832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,768,0.015485866864522298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,512,0.015570132931073507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,64,0.01492800017197927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,128,0.015035733580589294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,65536,0.10455893675486247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,32,0.01520853340625763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,10240,0.030194133520126343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,8192,0.0254037340482076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,7168,0.024209066232045492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,16384,0.03713066577911377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,12288,0.03259306748708089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,6144,0.023180800676345825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,5120,0.021929599841435752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,1536,0.016280532876650492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,4096,0.020413867632548013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,3584,0.020117332537968956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,3072,0.019347200791041054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,2048,0.017799466848373413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,2560,0.018684800465901694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,1024,0.015837867061297098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,512,0.015447466572125753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,768,0.015612799922625223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,256,0.015041066209475198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,32,0.015018666783968607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,128,0.01527253290017446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,64,0.015201066931088766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,65536,0.09509332974751791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,10240,0.02835306723912557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,8192,0.027075199286142985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,7168,0.025933865706125898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,6144,0.024446932474772136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,12288,0.030264532566070555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,16384,0.0350325345993042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,5120,0.021172267198562623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,4096,0.020091732343037925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,3584,0.019471999009450278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,1536,0.016058666507403056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,3072,0.018998400370279948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,2560,0.01798186699549357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,2048,0.016334933042526246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,512,0.015424000223477683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,1024,0.016032000382741295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,768,0.015836800138155617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,256,0.015225600202878317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,128,0.014855466286341348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,65536,0.0839850664138794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,64,0.01506239970525106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,32,0.015033599734306336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,10240,0.026524800062179565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,8192,0.024951465924580894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,6144,0.023012266556421915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,12288,0.028485333919525145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,7168,0.023907200495402018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,16384,0.03205546736717224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,3584,0.01884053349494934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,5120,0.020321067174275717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,4096,0.019409066438674925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,3072,0.018201599518458046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,2560,0.016588800152142844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,2048,0.016300800442695617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,1536,0.015961600343386333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,512,0.015450666348139444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,1024,0.01560426652431488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,768,0.015549866358439126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,65536,0.07178880373636881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,64,0.014946132898330688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,256,0.01514240006605784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,32,0.015020799636840821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,128,0.015255467096964518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,12288,0.026053333282470705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,16384,0.028742400805155437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,7168,0.022923733790715536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,10240,0.02488320072491964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,8192,0.023611734310785927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,6144,0.02232426603635152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,5120,0.01990613341331482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,3584,0.018410666783650716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,4096,0.01904639999071757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,3072,0.016676266988118492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,2560,0.016337066888809204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,2048,0.016547200083732606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,65536,0.06206186612447103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,1024,0.015738667050997416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,1536,0.01625386675198873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,512,0.01528320014476776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,768,0.015543466806411744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,128,0.015204266707102457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,32,0.015020799636840821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,256,0.015150933464368185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,64,0.014841600259145101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,16384,0.026358399788538617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,12288,0.024371200799942018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,8192,0.022194133202234904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,10240,0.02341866691907247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,6144,0.021193599700927733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,7168,0.021772799889246623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,5120,0.019444266955057778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,4096,0.01734613378842672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,3584,0.01732800006866455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,65536,0.05160319805145264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,3072,0.017157334089279174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,1536,0.015921066204706825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,2560,0.016361600160598753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,2048,0.01634666621685028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,1024,0.015715199708938598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,768,0.015543466806411744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,128,0.014920533696810404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,512,0.015382400155067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,256,0.015316266814867655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,64,0.014802133043607077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,32,0.014969600240389505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,16384,0.02365866700808207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,12288,0.02204586664835612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,8192,0.02058239976565043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,10240,0.021542400121688843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,7168,0.0191648006439209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,65536,0.040402134259541825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,6144,0.01914026737213135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,4096,0.01770346760749817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,5120,0.018252799908320107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,3584,0.017528533935546875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,2560,0.016572800278663636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,3072,0.01715946594874064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,1024,0.015542399883270264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,2048,0.01646719972292582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,768,0.015500799814860026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,1536,0.01602773368358612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,512,0.015545599659283958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,64,0.015210666259129844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,256,0.014954666296641031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,128,0.0155349334081014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,32,0.015336533387502035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,16384,0.022566399971644082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,10240,0.020214400688807168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,12288,0.02108906706174215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,8192,0.018634666999181114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,7168,0.01925546725591024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,65536,0.034916265805562334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,6144,0.01922986706097921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,4096,0.01760960022608439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,5120,0.01839039921760559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,3584,0.017433599630991618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,3072,0.01691626707712809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,1024,0.015571199854214988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,2048,0.016403200229008992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,1536,0.015972266594568886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,2560,0.016775466998418174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,768,0.015786666671435037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,512,0.01566933294137319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,128,0.014988799889882406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,256,0.015098667144775391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,32,0.015050666530927024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,64,0.01511146624883016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,10240,0.01944533387819926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,12288,0.01880000034968058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,16384,0.02090453306833903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,65536,0.029682133595148725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,8192,0.018709333737691243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,5120,0.018515199422836304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,7168,0.01926400065422058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,6144,0.01915093262990316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,4096,0.017898666858673095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,3584,0.017497599124908447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,3072,0.017308799425760905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,2048,0.01649173299471537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,2560,0.01663253307342529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,1536,0.01602026621500651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,1024,0.01585706671079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,768,0.015717333555221556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,128,0.014980266491572062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,256,0.015244799852371215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,512,0.015373866756757101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,64,0.014854400356610616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,32,0.014963199694951376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,65536,0.024052266279856363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,12288,0.01895573337872823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,16384,0.019387733936309815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,10240,0.019476266702016194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,7168,0.019450666507085164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,6144,0.01912213365236918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,8192,0.01893226703008016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,4096,0.01773759921391805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,5120,0.0184661328792572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,3584,0.017451733350753784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,1024,0.015640532970428465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,2560,0.016578132907549538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,3072,0.01716586748758952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,2048,0.01646613379319509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,768,0.015533866484959922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,1536,0.01596799989541372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,256,0.015051733454068503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,512,0.01548479994138082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,128,0.014798933267593383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,32,0.015010133385658264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,64,0.015165866414705912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,65536,0.021513599157333373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,16384,0.019090133905410766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,12288,0.018629332383473717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,8192,0.01869973341623942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,10240,0.01922986706097921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,7168,0.019283199310302736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,3584,0.017017600933710735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,6144,0.018895999590555827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,5120,0.017997866868972777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,4096,0.017374932765960693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,3072,0.016666666666666666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,1536,0.01578879952430725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,2048,0.016241066654523215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,2560,0.01657919983069102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,1024,0.015824000040690102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,768,0.015412267049153647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,512,0.015192533532778421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,128,0.014757333199183145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,256,0.014878933628400167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,64,0.014968533317248026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,32,0.01513920029004415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,10240,0.018979199727376304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,65536,0.019990400473276774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,16384,0.0191210667292277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,12288,0.018772266308466592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,8192,0.018580265839894614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,5120,0.01790613333384196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,7168,0.01924906571706136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,6144,0.01891626715660095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,4096,0.017525333166122436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,2048,0.016103466351826988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,2560,0.016345600287119545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,3072,0.016693333784739174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,3584,0.017127466201782227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,1536,0.015894400080045064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,512,0.015224533279736838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,1024,0.01569066643714905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,768,0.015383467078208923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,256,0.01492586632569631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,128,0.014914133151372275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,32,0.014940800269444785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,64,0.014921599626541137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,65536,0.020039467016855876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,16384,0.019091200828552247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,10240,0.018990933895111084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,12288,0.01881813406944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,5120,0.01791999936103821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,7168,0.019233065843582153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,8192,0.01853653391202291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,6144,0.01877973278363546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,2560,0.016428800423940022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,3584,0.017011199394861856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,4096,0.017518933614095053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,3072,0.01684479912122091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,2048,0.016123732924461363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,768,0.015401599804560342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,1536,0.015675733486811318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,1024,0.015585066874821982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,512,0.015004799763361613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,32,0.014828800161679586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,256,0.01509119967619578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,64,0.014823466539382935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,128,0.014904533823331198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,5120,0.12689812978108722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,6144,0.14895893732706705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,7168,0.17146239280700684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,8192,0.195303471883138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,10240,0.23711466789245605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,4096,0.10329066912333171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,2560,0.07170666853586832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,3584,0.09358826478322348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,3072,0.08284693559010824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,12288,0.28303467432657875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,256,0.02241493264834086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,768,0.03282986680666606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,1024,0.03796906471252441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,512,0.026947200298309326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,128,0.021253333489100138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,1536,0.04967360099156697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,2048,0.06360426743825277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,64,0.019618133703867592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,32,0.01916266679763794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,16384,0.3755242665608724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,7168,0.05347413221995036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,8192,0.058974933624267575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,10240,0.06946986516316732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,12288,0.0807925303777059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,6144,0.05459093252817789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,5120,0.0426090677579244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,16384,0.10173226992289226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,4096,0.03737279971440633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,3584,0.03470613161722819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,3072,0.03213013410568237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,1536,0.023602133989334105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,2048,0.026447999477386474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,1024,0.02094399929046631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,2560,0.029389866193135578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,768,0.019362133741378785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,256,0.01590933303038279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,512,0.018234666188557944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,128,0.015429332852363586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,64,0.0155157337586085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,32,0.015361066659291586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,10240,0.05678613185882568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,8192,0.048884268601735434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,12288,0.065611732006073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,7168,0.044609065850575766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,16384,0.08094933032989501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,6144,0.040522666772206624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,5120,0.036218667030334474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,3584,0.029997867345809937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,4096,0.03213226596514384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,3072,0.027977599700291948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,2560,0.025587199131647746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,2048,0.02367466688156128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,1536,0.021861332654953002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,1024,0.019795199235280357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,512,0.015875200430552162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,768,0.018556799491246542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,256,0.01581546664237976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,32,0.015387733777364096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,128,0.015430399775505066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,64,0.015620266397794088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,65536,0.35716800689697265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,12288,0.05752426783243815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,8192,0.04341013431549072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,10240,0.0492789347966512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,16384,0.07000319957733155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,6144,0.036526934305826826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,5120,0.03318079908688863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,7168,0.043933868408203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,4096,0.029233066240946452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,3584,0.02751893401145935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,3072,0.025763199726740522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,2560,0.023966934283574423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,768,0.017090133825937905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,2048,0.022373332579930624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,1024,0.01874666611353556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,1536,0.02059626579284668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,65536,0.27769387563069664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,256,0.01539306640625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,512,0.01590506633122762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,128,0.015335466464360556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,64,0.015192533532778421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,32,0.015170133113861084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,8192,0.038438399632771805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,16384,0.058234667778015135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,7168,0.03480106592178345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,10240,0.04337386687596639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,12288,0.04862720171610514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,65536,0.23560214042663574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,3072,0.023639466365178427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,3584,0.02516053318977356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,5120,0.02927680015563965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,6144,0.03206400076548259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,4096,0.026342399915059406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,2560,0.02386346658070882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,2048,0.020962133010228475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,256,0.0155157337586085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,1024,0.01770346760749817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,1536,0.01966080069541931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,768,0.015710933009783427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,512,0.015800533692042033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,128,0.01535146633783976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,64,0.015288533767064414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,32,0.0151936004559199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,7168,0.03265173236529033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,8192,0.03476693232854207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,12288,0.047031466166178384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,10240,0.04246293306350708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,16384,0.05657066504160563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,65536,0.1891637325286865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,5120,0.027784534295399982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,6144,0.030306132634480794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,4096,0.02515520056088765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,3072,0.022875734170277915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,3584,0.02421226700146993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,2560,0.022706133127212525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,2048,0.02031466762224833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,1536,0.019129600127538046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,768,0.015867732961972556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,1024,0.016115199526151022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,512,0.015664000312487283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,256,0.015212800105412802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,128,0.015186132987340293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,64,0.015096533298492431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,32,0.015178666512171427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,8192,0.03326293428738912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,10240,0.03745919863382975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,7168,0.031058132648468018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,16384,0.04953066507975261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,12288,0.04147946834564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,65536,0.17181013425191244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,5120,0.026589866479237872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,4096,0.02453440030415853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,6144,0.02887360056241353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,3072,0.021929599841435752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,3584,0.023312000433603923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,2560,0.022091732422510783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,2048,0.019642666975657145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,1536,0.018550399939219156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,1024,0.015824000040690102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,768,0.015893333156903586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,512,0.015877333283424378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,128,0.015119999647140503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,256,0.015380266308784484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,64,0.015124266346295675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,32,0.01503679951032003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,12288,0.036600534121195474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,10240,0.033346132437388105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,65536,0.14953279495239258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,7168,0.02767360011736552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,8192,0.0293503999710083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,16384,0.04350293477376302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,3584,0.021556266148885093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,6144,0.02605973283449809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,5120,0.02434239983558655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,3072,0.02071146567662557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,4096,0.022309333086013794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,2560,0.020039467016855876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,2048,0.01885120073954264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,1536,0.01732906699180603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,768,0.01554026703039805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,1024,0.016301866372426352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,512,0.015447466572125753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,256,0.015388799707094827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,128,0.014959999918937683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,64,0.015195733308792115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,32,0.015083733201026916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,65536,0.12203093369801839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,10240,0.031883732477823896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,7168,0.0254314661026001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,12288,0.03469333251317342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,8192,0.029014400641123456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,16384,0.040072532494862874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,6144,0.023869866132736207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,5120,0.022613332668940226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,3072,0.01990293264389038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,4096,0.02133013407389323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,3584,0.02067199945449829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,2560,0.01914026737213135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,1024,0.015892266233762106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,1536,0.016385066509246825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,2048,0.018105600277582803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,768,0.015729066729545594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,512,0.015602133671442666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,256,0.0151936004559199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,128,0.015059199929237366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,64,0.015013333161671957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,32,0.014886400103569031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,65536,0.10449813206990559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,8192,0.025462400913238526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,7168,0.02414399981498718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,6144,0.023038933674494423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,16384,0.03717973232269287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,10240,0.030003199974695843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,12288,0.032270934184392294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,5120,0.021849600474039714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,4096,0.020436267058054604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,1536,0.01627840002377828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,3072,0.019427200158437095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,2560,0.018809600671132406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,3584,0.020087466637293497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,2048,0.01765973369280497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,1024,0.0156768004099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,768,0.015562666455904641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,256,0.01525759994983673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,512,0.01562773287296295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,128,0.015060266852378846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,65536,0.09510400295257568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,64,0.014975999792416891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,32,0.014866133530934652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,8192,0.027020800113677978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,16384,0.03426133394241333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,10240,0.028296534220377607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,7168,0.025779199600219727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,6144,0.024361600478490196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,12288,0.030459733804066975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,4096,0.019986132780710854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,5120,0.02126400073369344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,3584,0.01957119901974996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,2560,0.01722666621208191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,3072,0.018990933895111084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,2048,0.01680319905281067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,1536,0.016193067034085594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,1024,0.015876266360282897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,65536,0.08195199966430664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,64,0.015153066317240397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,512,0.01529813309510549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,768,0.015811199943224587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,256,0.015121066570281982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,128,0.015282133221626281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,32,0.015090133746465048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,16384,0.03183146715164185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,12288,0.028164267539978027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,10240,0.026549333333969118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,8192,0.024583466847737632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,7168,0.024116265773773193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,6144,0.02328959902127584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,5120,0.020502400398254395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,4096,0.019403733809789023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,3584,0.01911679903666178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,3072,0.01737173398335775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,2560,0.016727467377980552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,1536,0.016122666994730632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,2048,0.01590933303038279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,1024,0.015564800302187601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,65536,0.07157546679178874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,128,0.015068800250689188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,768,0.015756799777348836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,64,0.014934399724006652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,256,0.01532266636689504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,512,0.015356799960136414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,32,0.014838400483131408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,16384,0.028777599334716797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,10240,0.02490560015042623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,12288,0.026045866807301837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,8192,0.02352959911028544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,7168,0.022772266467412313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,6144,0.02208426594734192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,5120,0.01989013353983561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,4096,0.01884160041809082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,3584,0.017642666896184288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,3072,0.016927999258041383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,1536,0.016025599837303162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,2560,0.016821332772572837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,65536,0.061603200435638425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,2048,0.016201600432395935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,1024,0.015682133038838704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,256,0.015135999520619711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,64,0.01487573285897573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,512,0.015553067127863566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,768,0.015711999932924905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,128,0.015013333161671957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,32,0.014800000190734863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,16384,0.02630293369293213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,12288,0.02412160038948059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,10240,0.023449599742889404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,8192,0.02217280069986979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,6144,0.021104000012079873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,7168,0.021741867065429688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,5120,0.01958720088005066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,4096,0.01730666756629944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,65536,0.050824534893035886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,3584,0.017172267039616905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,2560,0.016495999693870545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,3072,0.017322667439778647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,2048,0.01634239951769511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,1024,0.015620266397794088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,1536,0.016057599584261575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,512,0.015532799561818442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,768,0.01550933321317037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,32,0.014934399724006652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,256,0.014974932869275412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,128,0.015227733055750528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,64,0.014980266491572062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,10240,0.021642667055130006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,12288,0.021996800104777017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,8192,0.020486400524775187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,16384,0.023596799373626708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,7168,0.020438400904337566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,65536,0.04010133345921834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,6144,0.018898133436838785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,5120,0.018427733580271402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,4096,0.01771946748097738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,3584,0.01720106601715088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,1024,0.015592533349990844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,3072,0.017256534099578856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,2560,0.016922666629155477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,2048,0.01637226641178131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,1536,0.01603626708189646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,256,0.01504746675491333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,768,0.015482667088508605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,512,0.015171200037002563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,128,0.015526400009791056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,64,0.015478400389353433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,32,0.015367466211318969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,16384,0.022292266289393105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,12288,0.020963199933369956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,10240,0.020360533396402994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,6144,0.01920426686604818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,8192,0.018911999464035035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,65536,0.03507519960403442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,5120,0.01840106646219889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,7168,0.019332265853881835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,4096,0.017691733439763387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,3584,0.017199999094009398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,1536,0.015947733322779337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,3072,0.017010132471720375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,2560,0.01694186727205912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,2048,0.016429866353670754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,1024,0.01597119967142741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,768,0.015793066223462424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,256,0.015144532918930054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,512,0.015316266814867655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,128,0.015156267086664834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,64,0.015290666619936624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,32,0.015015467007954916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,12288,0.01918506622314453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,16384,0.020805333058039346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,10240,0.0192522664864858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,8192,0.01909866730372111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,65536,0.029370667537053426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,7168,0.019438934326171876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,5120,0.01839573383331299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,6144,0.018994132677714028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,4096,0.01771946748097738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,3072,0.017198934157689413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,3584,0.01728960076967875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,2560,0.017015467087427773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,1536,0.01595093309879303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,2048,0.01665600041548411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,512,0.015478400389353433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,768,0.015496533115704855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,1024,0.015786666671435037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,256,0.014969600240389505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,64,0.014879999558130899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,128,0.015166933337847391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,32,0.015173332889874777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,16384,0.018947199980417887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,65536,0.023910399278004965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,8192,0.01872533361117045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,10240,0.019019732872645058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,12288,0.01885546644528707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,7168,0.019509333372116088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,6144,0.019059199094772338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,3584,0.017283199230829875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,4096,0.017767467101415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,5120,0.01834453344345093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,3072,0.01688533425331116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,2560,0.016663466890652977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,1024,0.015582933028539022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,1536,0.016195199886957803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,2048,0.016523733735084534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,768,0.015516799688339234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,256,0.014889599879582724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,512,0.015358933806419372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,64,0.01486186683177948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,128,0.014854400356610616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,32,0.015101866920789084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,16384,0.019077332814534505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,65536,0.0213536004225413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,12288,0.018682666619618735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,10240,0.019100799163182577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,8192,0.01858133276303609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,7168,0.01922773321469625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,5120,0.01802133321762085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,6144,0.018911999464035035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,2560,0.01639573375384013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,3584,0.017042134205500284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,3072,0.01688533425331116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,4096,0.017592533429463705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,2048,0.016025599837303162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,1536,0.01592853367328644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,768,0.015404799580574035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,1024,0.015804800391197204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,512,0.015059199929237366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,256,0.015103999773661295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,128,0.015100799997647605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,65536,0.019735467433929444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,32,0.014940800269444785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,64,0.014899200201034546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,16384,0.019014400243759156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,12288,0.01856000026067098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,10240,0.0190720001856486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,7168,0.019144533077875774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,8192,0.018832000096638997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,6144,0.01880533297856649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,5120,0.017893334229787193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,3584,0.01700906753540039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,4096,0.017523199319839478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,3072,0.016846932967503867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,2560,0.016448000073432924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,1536,0.015809067090352378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,2048,0.016280532876650492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,768,0.015251200397809347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,1024,0.015341867009798685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,256,0.015130666891733804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,128,0.014776532848676046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,512,0.015434666474660238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,64,0.014973866939544677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,32,0.015105066696802774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,7168,0.019081600507100425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,65536,0.019845332702000937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,10240,0.018908800681432088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,12288,0.018764799833297728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,16384,0.018965333700180054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,8192,0.01879253387451172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,6144,0.01895893414815267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,5120,0.018017067511876424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,4096,0.017271467049916587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,3584,0.01699733336766561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,3072,0.01685546636581421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,2560,0.016277333100636802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,2048,0.016246400276819863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,1024,0.015593600273132325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,1536,0.01593280037244161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,512,0.015153066317240397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,768,0.01532799998919169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,256,0.015093333522478738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,32,0.0149536003669103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,128,0.014751999576886495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,64,0.014893866578737893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,5120,4.8801417032877605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,6144,5.886277262369791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,7168,6.918916320800781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,8192,7.681517028808594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,10240,9.8779296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,4096,3.852093760172526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,12288,11.949037679036458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,3584,3.3533589680989584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,3072,2.971905008951823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,2560,2.5303072611490887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,2048,2.114329655965169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,256,0.7763402938842774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,512,0.9033482869466146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,1536,1.72040532430013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,768,1.1206954956054687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,1024,1.249607467651367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,128,0.7077077229817708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,16384,15.931769816080728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,7168,1.6651220957438153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,8192,1.8887189229329426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,6144,1.4092234293619792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,10240,2.5362826029459637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,12288,2.910832977294922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,16384,4.03376210530599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,4096,0.949072011311849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,3584,0.8510133107503256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,5120,1.1745780944824218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,3072,0.7458688100179036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,2560,0.6236234664916992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,256,0.20248853365580238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,512,0.23997653325398766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,768,0.3040501276652018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,1024,0.33470293680826824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,1536,0.4352448145548503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,2048,0.5496906916300456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,128,0.18472320238749187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,7168,1.230188751220703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,8192,1.431216049194336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,10240,1.7952746073404948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,12288,2.1433855692545576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,16384,2.9436480204264326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,5120,0.9035711924235026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,4096,0.7312405268351237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,6144,1.0522794723510742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,3584,0.6525610605875651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,2560,0.4902336120605469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,2048,0.4122954686482747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,3072,0.5613152186075847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,1536,0.3348842620849609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,1024,0.253436803817749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,128,0.14101227124532062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,256,0.15510613123575848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,512,0.18621546427408855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,768,0.22835413614908853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,8192,1.2154655456542969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,10240,1.483621342976888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,12288,1.7474868774414063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,65536,16.354953002929687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,16384,2.4633941650390625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,6144,0.9105365117390951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,7168,1.0617685317993164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,5120,0.7476245244344075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,4096,0.6172789255777995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,3584,0.5476298650105794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,1536,0.2849909464518229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,2560,0.41453866958618163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,2048,0.34608532587687174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,3072,0.4783647855122884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,1024,0.21364906628926597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,256,0.13112213611602783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,512,0.15686933199564618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,128,0.1185802698135376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,768,0.19268053372701008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,65536,13.295451863606772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,8192,0.9793408075968424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,10240,1.230392583211263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,12288,1.4908042907714845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,16384,1.979849624633789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,7168,0.8561909357706705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,5120,0.6175082524617512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,6144,0.7426261266072591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,4096,0.5189471880594889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,3584,0.45765120188395186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,3072,0.3949066797892252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,2560,0.33509120941162107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,2048,0.290775458017985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,1536,0.23803413709004723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,512,0.1289578676223755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,256,0.10802453358968098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,128,0.09806186358133952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,1024,0.17934187253316242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,768,0.15907626152038573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,65536,10.146035766601562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,7168,0.7674026489257812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,8192,0.8661120096842447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,10240,1.082514190673828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,12288,1.3431583404541017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,16384,1.7597632090250652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,4096,0.4575477282206218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,5120,0.5654624303181965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,6144,0.6714954376220703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,3584,0.41850026448567706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,3072,0.3586282730102539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,2560,0.3031701405843099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,1536,0.2144789377848307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,2048,0.25848639806111656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,1024,0.16518826484680177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,128,0.0883903980255127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,512,0.11873706976572673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,256,0.09783679644266764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,768,0.14594133694966632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,65536,8.480436197916667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,8192,0.76505921681722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,10240,0.9463082631429037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,7168,0.6670421600341797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,12288,1.1082784016927083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,16384,1.5605365753173828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,5120,0.4872597376505534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,4096,0.4015018781026204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,3584,0.3559722582499186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,6144,0.5801631927490234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,2560,0.26645013491312664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,3072,0.3098858515421549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,1024,0.1412384033203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,1536,0.1837109406789144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,2048,0.22586347262064616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,512,0.10088000297546387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,128,0.0763861338297526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,768,0.12391786575317383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,256,0.08394239743550619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,65536,7.508996073404949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,8192,0.6507402420043945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,10240,0.8152565638224283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,7168,0.5781312306722005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,12288,0.9655018488566081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,16384,1.3074698130289713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,5120,0.4210272153218587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,6144,0.5057120005289714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,4096,0.3471829414367676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,3584,0.3097920099894206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,2560,0.23186880747477212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,3072,0.27085653940836585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,2048,0.1953013261159261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,1536,0.1614496072133382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,1024,0.12384426593780518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,128,0.06554986635843912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,256,0.07224533557891846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,512,0.08846826553344726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,768,0.10860053698221843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,65536,6.634285990397136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,8192,0.5446858723958333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,7168,0.48641281127929686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,10240,0.6912511825561524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,12288,0.8116949081420899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,16384,1.1141834259033203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,5120,0.3549450556437174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,4096,0.2895381291707357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,6144,0.4172181447347005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,3584,0.2570112069447835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,2560,0.19454186757405598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,3072,0.2247231960296631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,2048,0.16223680178324382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,1536,0.13236693541208905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,1024,0.10134080251057942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,128,0.0538538654645284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,256,0.05876906712849935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,512,0.07282773653666177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,768,0.0892352024714152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,65536,5.367742919921875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,7168,0.4366880098978679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,8192,0.5014399846394857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,10240,0.6242176055908203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,12288,0.7427839914957682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,16384,0.9708778381347656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,5120,0.3263434727986654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,4096,0.26358613967895506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,6144,0.3822005271911621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,3584,0.23896533648173013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,3072,0.20631465911865235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,2560,0.17840746243794758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,2048,0.14867199261983235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,1536,0.12102719942728679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,1024,0.0936405340830485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,128,0.048265600204467775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,256,0.053965866565704346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,512,0.06599040031433105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,768,0.08137386639912923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,65536,4.684808349609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,7168,0.38750505447387695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,8192,0.4387701352437337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,10240,0.5585184097290039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,12288,0.6550389607747396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,16384,0.8527850468953451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,5120,0.2834474563598633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,4096,0.23488640785217285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,6144,0.33753814697265627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,3584,0.2106346607208252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,2560,0.15773866971333822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,3072,0.1824127991994222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,2048,0.1326090653737386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,1536,0.10777386824289958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,1024,0.08427626291910807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,128,0.0427072008450826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,256,0.047065599759419756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,512,0.058867200215657556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,768,0.0723306655883789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,65536,4.256931304931641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,7168,0.3428586641947428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,8192,0.38644161224365237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,10240,0.4831424077351888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,12288,0.5828298568725586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,16384,0.7557685216267903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,5120,0.2511616071065267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,4096,0.2045087973276774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,6144,0.2971477190653483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,3584,0.18557440439860026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,2560,0.13835733731587727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,3072,0.16051093737284344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,2048,0.11622400283813476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,1536,0.09530239899953206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,1024,0.0730133295059204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,128,0.03698773384094238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,256,0.04092373450597127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,512,0.050920534133911136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,768,0.0630741318066915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,65536,3.7653129577636717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,8192,0.33365119298299156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,10240,0.4106101353963216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,7168,0.296015993754069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,12288,0.4886752128601074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,16384,0.657213846842448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,4096,0.1787989298502604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,5120,0.21359893480936684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,6144,0.25524800618489585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,3584,0.157369597752889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,3072,0.13807679812113444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,2560,0.11947519779205322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,2048,0.09857920010884604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,1536,0.07989973227183024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,1024,0.061921068032582606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,256,0.03399360179901123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,512,0.04269546667734782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,768,0.05380906661351522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,128,0.030509867270787555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,65536,3.2228459676106773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,7168,0.2517504056294759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,8192,0.2827103932698568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,10240,0.3550304094950358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,12288,0.4215765317281087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,16384,0.5443338394165039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,5120,0.18549866676330568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,4096,0.15116480191548665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,6144,0.21810879707336425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,3584,0.13676053682963055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,2560,0.10162133375803631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,3072,0.1175114631652832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,1536,0.06841599941253662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,2048,0.08361919720967612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,1024,0.05275306701660156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,128,0.024206932385762533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,256,0.028217599789301558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,512,0.03585280179977417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,65536,2.751912434895833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,768,0.04478293259938558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,7168,0.2052768071492513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,8192,0.23539519309997559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,10240,0.2859765370686849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,12288,0.34353920618693035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,16384,0.4474090576171875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,5120,0.1525045394897461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,6144,0.17838932673136393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,4096,0.12257493336995443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,3584,0.11061973571777343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,3072,0.09504213333129882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,2560,0.08158720334370931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,1536,0.05465386708577474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,2048,0.06775573094685873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,1024,0.04225279887517293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,128,0.0179967999458313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,256,0.020878932873407998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,768,0.0352021336555481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,512,0.02749546567598979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,65536,2.233807881673177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,7168,0.18286293347676594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,8192,0.2099957307179769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,10240,0.25426026980082195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,12288,0.30193386077880857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,16384,0.39909868240356444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,5120,0.1351093292236328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,4096,0.1087498664855957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,6144,0.1583669344584147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,3584,0.09632960160573324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,3072,0.08381333351135253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,2560,0.07207039992014566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,2048,0.06008426745732626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,1536,0.048603733380635575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,1024,0.03660159905751546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,128,0.015275733669598899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,256,0.017794134219487508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,512,0.023516800006230673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,768,0.030370134115219116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,65536,1.766823450724284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,8192,0.1829642613728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,7168,0.16273172696431476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,10240,0.2286901315053304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,12288,0.2683168093363444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,16384,0.35296961466471355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,5120,0.11748373508453369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,4096,0.09648533662160239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,6144,0.13958187103271485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,3584,0.0846506675084432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,3072,0.0738090674082438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,2560,0.062882133324941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,2048,0.052401065826416016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,1536,0.04158186515172323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,1024,0.030320000648498536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,65536,1.588360595703125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,256,0.0146506667137146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,128,0.012222933769226074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,512,0.019509333372116088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,768,0.02488853335380554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,7168,0.14240320523579914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,8192,0.1608448028564453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,10240,0.19913279215494792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,12288,0.2394165356953939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,16384,0.31489280064900715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,5120,0.10230080286661784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,6144,0.12207039992014568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,4096,0.08324159781138102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,3584,0.07364479700724283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,1536,0.0347434679667155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,3072,0.06376213232676188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,2560,0.05367146730422974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,2048,0.04434773524602254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,1024,0.02430400053660075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,128,0.009278933207194012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,256,0.011378133296966552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,65536,1.3599168141682942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,512,0.015986133615175882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,768,0.02001813252766927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,7168,0.13708480199178058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,8192,0.15574080149332684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,10240,0.19232746760050456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,12288,0.22887786229451498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,16384,0.30221439997355143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,5120,0.09858346780141194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,6144,0.1179850657780965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,4096,0.0792949358622233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,3584,0.07001067002614339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,3072,0.06016106605529785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,2048,0.04111253420511882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,1024,0.02259733279546102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,2560,0.051121068000793454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,1536,0.03242133259773254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,768,0.01802240014076233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,512,0.014286933342615762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,128,0.008132266501585644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,256,0.01016533374786377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,65536,1.1879583994547525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,6144,2.866260274251302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,7168,3.3765749613444007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,8192,3.7113568623860678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,10240,5.127559407552083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,12288,5.892040506998698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,4096,1.9097503662109374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,5120,2.2915606180826824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,3584,1.6846794128417968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,3072,1.4040672302246093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,1024,0.6151594797770182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,16384,7.812243143717448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,2048,1.0359370549519857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,2560,1.1806196848551431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,1536,0.8302869160970052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,256,0.39164692560831704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,512,0.4514581362406413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,768,0.5707914352416992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,65536,1.1556660970052084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,128,0.35946451822916664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,6144,0.6999338785807292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,8192,0.9106880187988281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,7168,0.8206303914388021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,10240,1.1364395141601562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,12288,1.3388331095377604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,16384,1.8345066070556642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,4096,0.48732906977335616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,3584,0.43692372639973953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,5120,0.5885909398396809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,3072,0.37132374445597327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,2560,0.31644694010416663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,1536,0.22710399627685546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,1024,0.17399360338846842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,2048,0.2759519894917806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,768,0.15713493029276532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,512,0.12334187030792236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,256,0.10497173468271892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,128,0.09846186637878418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,10240,0.8853343963623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,8192,0.691425069173177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,7168,0.6170954386393229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,12288,1.0478357315063476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,16384,1.4658250172932943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,4096,0.3724639892578125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,5120,0.45305919647216797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,6144,0.5384746551513672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,3584,0.3343178749084473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,1536,0.17522560755411784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,3072,0.28786773681640626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,1024,0.1329578638076782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,2560,0.24763733545939126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,2048,0.20958827336629232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,256,0.08170560201009115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,128,0.07603733539581299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,512,0.09743786652882894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,768,0.11886293093363445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,8192,0.5957589467366536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,10240,0.7412021636962891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,12288,0.8834688186645507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,16384,1.195103963216146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,65536,7.9962717692057295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,7168,0.5222357432047526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,6144,0.45352853139241533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,5120,0.3824735959370931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,3584,0.281823984781901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,3072,0.24395947456359862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,4096,0.3148714701334635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,2560,0.2108309268951416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,2048,0.18003519376118976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,1024,0.11325013637542725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,1536,0.14877333641052246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,768,0.10072960058848064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,128,0.06458026568094889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,512,0.08300373554229737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,256,0.06957973639170328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,8192,0.4857269287109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,10240,0.602887471516927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,12288,0.7231957117716472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,65536,6.097802734375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,16384,0.9800394694010416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,4096,0.262993065516154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,5120,0.3121280034383138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,6144,0.36997973124186195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,7168,0.4271733283996582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,3584,0.23022185961405434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,2048,0.14779626528422038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,2560,0.17474133173624676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,512,0.06798186302185058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,1536,0.12140479882558186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,3072,0.20307839711507164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,768,0.0829482634862264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,1024,0.09353919823964438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,256,0.057048531373341885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,128,0.05308906634648641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,65536,4.859455871582031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,8192,0.4318858782450358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,10240,0.5461066563924153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,7168,0.3792832056681315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,12288,0.6425525029500325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,16384,0.856651751200358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,4096,0.22899093627929687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,6144,0.3318943977355957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,3072,0.18201386133829753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,3584,0.2091562589009603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,5120,0.28020801544189455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,2560,0.157368532816569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,1024,0.08518933455149333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,2048,0.13205333550771076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,1536,0.11050773461659749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,512,0.06247040033340454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,768,0.07519146601359049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,128,0.04720853169759114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,256,0.05170559883117676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,65536,3.9046859741210938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,10240,0.4710015932718913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,8192,0.37562987009684246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,7168,0.33099838892618816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,12288,0.5619946797688802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,16384,0.7485888163248698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,4096,0.20109866460164388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,3584,0.17960106531778972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,5120,0.24707093238830566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,6144,0.29151039123535155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,3072,0.1562378724416097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,2560,0.13751360575358074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,1536,0.09510613282521566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,2048,0.11514986356099446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,1024,0.0741919994354248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,768,0.06508266528447469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,512,0.05371626615524292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,256,0.0446613351504008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,128,0.04129813512166341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,65536,3.723877207438151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,8192,0.3247957229614258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,12288,0.4796650568644206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,10240,0.41486399968465165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,16384,0.63023681640625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,6144,0.25265280405680335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,4096,0.17405014038085936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,5120,0.21040746370951333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,7168,0.29360853830973305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,3584,0.1565109411875407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,1024,0.06500480175018311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,2048,0.10071252981821696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,1536,0.08393173217773438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,2560,0.11915946801503499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,3072,0.13645866711934407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,768,0.057202132542928066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,512,0.04746986627578735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,256,0.03940586646397908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,128,0.0357312003771464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,65536,3.116826629638672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,8192,0.2704981486002604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,7168,0.24178454081217446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,10240,0.3463285446166992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,12288,0.4015189488728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,16384,0.534884262084961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,5120,0.17754346529642742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,6144,0.2084767977396647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,4096,0.14741013844807943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,3584,0.13014187018076578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,3072,0.11361599763234456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,2048,0.08358720143636068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,2560,0.09956479867299398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,1024,0.05376000006993612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,1536,0.06981759866078695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,768,0.04685866832733154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,512,0.03861653407414754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,65536,2.6377492268880207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,256,0.03269226749738057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,128,0.029661866029103597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,7168,0.21886293093363443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,8192,0.2489151954650879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,12288,0.3699551900227865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,10240,0.3127434730529785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,6144,0.19241600036621093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,16384,0.47658348083496094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,4096,0.13480532964070638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,5120,0.16359786987304686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,3584,0.12147413094838459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,3072,0.10472853183746338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,2560,0.09065279960632325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,2048,0.07655466397603353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,1536,0.06386559804280599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,1024,0.05014933347702026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,768,0.04357333183288574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,512,0.03566826581954956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,256,0.02956266601880391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,128,0.026756266752878826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,65536,2.179845428466797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,8192,0.221891196568807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,7168,0.19592533111572266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,10240,0.27802133560180664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,16384,0.42462507883707684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,12288,0.3212373415629069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,4096,0.11910506884257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,6144,0.17146347363789877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,5120,0.14482240676879882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,3584,0.10597973664601643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,3072,0.09212266604105632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,2048,0.06877013047536215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,1024,0.04406079848607381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,1536,0.056892800331115725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,768,0.038471468289693195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,2560,0.08051946957906088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,128,0.023337600628534953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,256,0.026097067197163898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,512,0.0320032000541687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,65536,2.038706080118815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,10240,0.24399785995483397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,8192,0.19623252550760906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,7168,0.17680106163024903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,12288,0.2869920094807943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,16384,0.37559467951456704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,5120,0.12815573215484619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,6144,0.14947412808736166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,3584,0.09381226698557535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,4096,0.1069600025812785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,3072,0.08126506805419922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,2048,0.060628267129262295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,1024,0.03931306600570679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,2560,0.07095359961191813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,128,0.020475733280181884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,768,0.033831465244293216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,1536,0.05034346580505371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,512,0.027543467283248902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,256,0.022567466894785563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,65536,1.782796859741211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,7168,0.1517632007598877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,8192,0.17223679224650065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,10240,0.20690560340881348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,12288,0.24839359919230142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,16384,0.327021853129069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,6144,0.13211839993794758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,4096,0.08951786359151205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,5120,0.11203946272532146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,3584,0.08010240395863852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,2560,0.06132053136825562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,3072,0.07006613413492838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,2048,0.05245440006256104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,1536,0.043398400147755936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,1024,0.03281919956207276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,512,0.023027199506759643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,768,0.028492800394694012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,256,0.01876586675643921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,128,0.01738986571629842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,65536,1.4849408467610679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,7168,0.12939306894938152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,8192,0.14722560246785482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,10240,0.18029866218566895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,12288,0.21367252667744957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,16384,0.2762197176615397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,4096,0.07709759871164958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,5120,0.09736746946970622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,6144,0.11223680178324383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,3584,0.0687946637471517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,3072,0.06041599909464518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,1024,0.02811093330383301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,2048,0.04519893328348796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,1536,0.037064532438913986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,2560,0.053082664807637535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,768,0.02388906677563985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,512,0.01995413303375244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,256,0.01600320041179657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,128,0.014705066879590353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,65536,1.2990880330403647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,7168,0.10571413040161133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,8192,0.12011733055114746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,10240,0.14843200047810873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,12288,0.17542719841003418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,16384,0.22696000734965005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,5120,0.07705919742584229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,6144,0.09171520074208578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,4096,0.06304106712341309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,3072,0.049201067288716635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,3584,0.05634133418401083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,768,0.019245866934458414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,2560,0.04345813194910685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,1024,0.02218773365020752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,2048,0.03647040128707886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,1536,0.029388799269994097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,65536,1.0882272084554037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,256,0.012772267063458761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,512,0.01595200002193451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,128,0.011714133620262145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,12288,0.16630080540974934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,7168,0.09776106675465902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,10240,0.14044586817423504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,8192,0.11445333162943523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,16384,0.21997119585673014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,65536,0.8732821146647135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,4096,0.05930240154266357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,3584,0.05304426749547323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,6144,0.08619093100229899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,5120,0.07183893521626791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,3072,0.045857067902882895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,512,0.013876266280810037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,1536,0.025996800263722735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,768,0.016642133394877114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,1024,0.020564266045888267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,2560,0.040727468331654866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,2048,0.033404799302419026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,128,0.00981119970480601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,256,0.01084266702334086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,8192,0.09341973463694254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,10240,0.11594026883443195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,12288,0.13557119369506837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,7168,0.08252053260803223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,16384,0.18216746648152668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,6144,0.0710378646850586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,3584,0.044299733638763425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,4096,0.04945173263549805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,5120,0.060154668490091955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,65536,0.8415221532185873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,3072,0.039060266812642415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,2048,0.02726400097211202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,2560,0.03360639810562134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,256,0.009603200356165568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,1536,0.021967999140421548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,1024,0.017072000106175742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,768,0.014612266421318054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,128,0.008740267157554627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,512,0.012193066875139873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,7168,0.07437120278676351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,8192,0.08427093029022217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,10240,0.104259196917216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,12288,0.1253493309020996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,16384,0.16348053614298502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,6144,0.06408640146255493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,5120,0.05410240093866984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,4096,0.044436268011728924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,3584,0.04037226835886638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,3072,0.034474666913350424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,65536,0.6874154408772786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,2048,0.024346667528152465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,2560,0.03006400068600972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,1536,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,768,0.012647466858228049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,1024,0.01502826710542043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,512,0.01058666706085205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,256,0.008457600076993307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,128,0.007713066538174947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,10240,0.1023360013961792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,8192,0.08264426390329996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,12288,0.12201493581136066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,7168,0.07287147045135497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,16384,0.1597610632578532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,65536,0.6284298578898112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,4096,0.04286613464355469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,5120,0.052510933081309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,3584,0.03840746482213338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,6144,0.06354879935582479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,3072,0.03288106719652812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,1024,0.014003200332323709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,512,0.009704533219337463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,2560,0.028244266907374065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,768,0.011739733815193176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,2048,0.02258239984512329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,1536,0.018489599227905273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,128,0.006734933455785115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,256,0.00758186678091685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,6144,1.364997355143229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,8192,1.82073491414388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,65536,0.604696528116862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,7168,1.586303965250651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,10240,2.3842623392740885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,12288,2.9012041727701825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,3584,0.8177344004313151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,3072,0.6994389216105144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,2560,0.5932853062947591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,4096,0.9201109568277994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,5120,1.0997792561848958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,2048,0.5156352043151855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,128,0.18441813786824543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,1536,0.41678613026936845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,512,0.23422187169392905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,256,0.2000586668650309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,768,0.29167038599650064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,1024,0.3159221331278483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,16384,3.9962677001953124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,7168,0.3999914805094401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,10240,0.5676693598429362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,8192,0.4517013231913249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,6144,0.3578250567118327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,12288,0.6790517171223958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,16384,0.8906474431355795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,4096,0.2434687932332357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,1536,0.11594346364339192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,5120,0.2938336054484049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,3072,0.19208639462788898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,2048,0.13888853391011555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,3584,0.21651946703592934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,2560,0.1655765374501546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,1024,0.09058026472727457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,768,0.08185813426971436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,256,0.05667093197504679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,512,0.06619946559270223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,128,0.05308800141016642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,8192,0.3578133265177409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,7168,0.3066816012064616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,10240,0.43572479883829757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,12288,0.512277348836263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,16384,0.7062890370686848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,5120,0.2241589387257894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,6144,0.2741525332132975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,4096,0.18411199251810711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,3584,0.16649492581685382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,2560,0.12555413246154784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,3072,0.14529813130696614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,2048,0.1080351988474528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,512,0.05166293382644653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,1536,0.08878933588663737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,1024,0.07027200063069662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,768,0.0619925340016683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,256,0.04393813212712606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,128,0.041018664836883545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,8192,0.2973717371622721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,12288,0.4341418584187825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,65536,4.201667277018229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,10240,0.3726293245951334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,16384,0.5830506642659505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,3584,0.14181866645812988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,7168,0.26237866083780925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,6144,0.22480853398640951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,5120,0.19287145932515462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,4096,0.16076159477233887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,2048,0.09118719895680746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,2560,0.10858133633931477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,3072,0.12381546497344971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,1536,0.07679466406504312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,1024,0.05938666661580404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,65536,2.93198725382487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,768,0.05315626859664917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,512,0.044171734650929766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,256,0.03819626569747925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,128,0.03550506830215454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,7168,0.21614933013916016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,10240,0.3017610549926758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,8192,0.24113066991170248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,12288,0.3558474540710449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,5120,0.15832853317260742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,16384,0.4828192075093587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,3584,0.1164629300435384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,6144,0.18638505935668945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,4096,0.13132906754811605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,3072,0.10215466817220051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,2560,0.08968640168507894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,1536,0.0639135996500651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,1024,0.0499669353167216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,2048,0.07540799776713053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,768,0.04419840176900228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,65536,2.5489728291829428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,512,0.03728640079498291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,256,0.031956267356872556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,128,0.02962239980697632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,8192,0.21701013247172035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,7168,0.19048746426900226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,10240,0.2694485346476237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,12288,0.3200885454813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,16384,0.42028052012125655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,6144,0.16694080034891765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,4096,0.11832746664683025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,5120,0.1413237412770589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,3584,0.10658026536305745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,2048,0.06846186319986979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,3072,0.09219306310017904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,2560,0.0805023988087972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,65536,2.0035018920898438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,1536,0.05755946636199951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,1024,0.04609386523564656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,256,0.029025065898895263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,768,0.04056213299433391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,512,0.033642665545145674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,128,0.02682773272196452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,10240,0.23372160593668617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,8192,0.18997440338134766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,12288,0.2805514653523763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,7168,0.17146347363789877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,16384,0.37431147893269856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,6144,0.14706560770670574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,5120,0.12408213615417481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,2560,0.07015466690063477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,4096,0.10361706415812175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,3584,0.0945525328318278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,3072,0.08060266971588134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,2048,0.06005546649297079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,65536,1.742258071899414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,128,0.023638399442036946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,1536,0.05050773223241171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,1024,0.039640533924102786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,768,0.0352288007736206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,512,0.029586132367451983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,256,0.02489173412322998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,7168,0.14501867294311524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,8192,0.1639776070912679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,10240,0.20911893844604493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,6144,0.12731306552886962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,12288,0.2432917277018229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,5120,0.10886080265045166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,16384,0.317137082417806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,3072,0.07032533486684164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,3584,0.07992533047993979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,4096,0.09091306527455648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,2560,0.061009065310160315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,2048,0.05266133149464926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,1536,0.044139734903971356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,65536,1.438698705037435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,128,0.01973759929339091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,1024,0.03492586612701416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,768,0.03099626700083415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,256,0.022107734282811483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,512,0.02611306707064311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,7168,0.12305386861165364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,8192,0.13838826815287272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,10240,0.17345919609069824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,12288,0.20277546246846517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,4096,0.07552639643351236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,16384,0.26826454798380533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,6144,0.10785173575083415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,5120,0.09093973636627198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,1536,0.037109335263570145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,3584,0.06682026386260986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,3072,0.05932159821192423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,2560,0.05128959814707438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,2048,0.04459413290023804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,65536,1.254473622639974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,256,0.018333866198857626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,1024,0.02922026713689168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,768,0.02553066611289978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,128,0.016757333278656007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,512,0.021406932671864828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,7168,0.1260639985402425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,12288,0.20698134104410806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,10240,0.17379627227783204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,8192,0.13969173431396484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,16384,0.27406721115112304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,65536,1.0474058787027993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,6144,0.10790719985961914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,4096,0.07358613014221191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,3072,0.05680960019429525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,5120,0.08997973601023355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,3584,0.06494826475779215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,2560,0.050977067152659095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,512,0.017374932765960693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,1536,0.0329205334186554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,256,0.014712533354759217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,768,0.021308799584706627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,1024,0.026758400599161784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,2048,0.04148053328196208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,128,0.012596266468365988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,10240,0.14100799560546876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,65536,1.090127944946289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,8192,0.11519893010457356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,16384,0.21507627169291177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,12288,0.16567467053731283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,7168,0.10069973468780517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,4096,0.06158506472905477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,3584,0.055320533116658535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,6144,0.08638613224029541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,5120,0.07432213624318441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,2048,0.036497068405151364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,3072,0.04861013491948445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,2560,0.042555733521779375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,768,0.02133013407389323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,1024,0.02415999968846639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,1536,0.030635732412338256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,512,0.017915733655293784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,256,0.015387733777364096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,128,0.013979732990264893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,12288,0.1751541296641032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,7168,0.10201600392659504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,8192,0.11763733228047688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,10240,0.1531701405843099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,16384,0.22100906372070311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,65536,0.8543829600016275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,6144,0.09049279689788818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,4096,0.06124053398768107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,3072,0.04601600170135498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,3584,0.055462400118509926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,5120,0.07506559689839681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,2560,0.04109653234481812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,2048,0.03441919883092244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,1536,0.026710400978724163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,512,0.013981866836547851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,256,0.011789866288503011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,768,0.017122133572896322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,1024,0.021562665700912476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,128,0.010150399804115296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,65536,0.898689079284668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,12288,0.12727999687194824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,16384,0.1669119993845622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,8192,0.08643946647644044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,10240,0.10828800201416015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,7168,0.07627092997233073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,5120,0.056201601028442384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,6144,0.0664629340171814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,4096,0.047430400053660074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,3584,0.04290239810943604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,3072,0.037843199570973714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,2560,0.03271039923032125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,2048,0.028032000859578448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,256,0.012086400389671325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,1536,0.023489065965016685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,1024,0.018402133385340372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,768,0.016310399770736693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,512,0.013991467157999673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,128,0.01095466713110606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,12288,0.11649706363677978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,10240,0.09824106693267823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,7168,0.06949546337127685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,8192,0.08162773450215657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,16384,0.1547818660736084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,65536,0.636793581644694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,6144,0.06000746488571167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,4096,0.043271466096242266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,3072,0.03361920118331909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,3584,0.039087998867034915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,2560,0.029686399300893146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,5120,0.05302293300628662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,256,0.01018453339735667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,1536,0.020131200551986694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,2048,0.025329067309697466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,512,0.011878400047620138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,768,0.013751467068990072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,1024,0.016323199868202208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,128,0.008809600273768108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,8192,0.06050986846288046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,65536,0.5918634414672852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,10240,0.07449066638946533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,12288,0.08856960137685141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,16384,0.1165280024210612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,4096,0.034007465839385985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,3584,0.02990613381067912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,7168,0.053852800528208414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,3072,0.026679466168085735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,5120,0.04096746842066447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,2560,0.023492266734441124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,6144,0.04717013438542684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,2048,0.019858133792877198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,512,0.010380799571673077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,128,0.008077866832415263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,256,0.009136000275611877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,1024,0.013314132889111837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,1536,0.016482133666674295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,768,0.011876266201337178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,8192,0.0578325351079305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,7168,0.0509002685546875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,10240,0.07063999970753988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,12288,0.08410987059275309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,16384,0.1115008036295573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,5120,0.039061331748962404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,6144,0.04416853189468384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,4096,0.03130666613578796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,3072,0.02430293361345927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,3584,0.02814720074335734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,65536,0.441267204284668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,2560,0.022048000494639078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,2048,0.018273067474365235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,768,0.01060693363348643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,1024,0.012275200088818867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,1536,0.015241600076357522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,512,0.009353599945704142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,256,0.008089600006739299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,128,0.007034666836261749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,65536,0.42711998621622727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,8192,0.05290346542994181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,10240,0.06631999810536703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,12288,0.07704532941182454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,7168,0.04711039861043294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,16384,0.10073599815368653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,4096,0.028706133365631104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,5120,0.035412267843882246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,6144,0.04123946825663249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,3072,0.022428800662358604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,3584,0.02590399980545044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,2560,0.02001813252766927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,512,0.008700799942016602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,768,0.009954133629798889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,256,0.007909333209196727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,2048,0.016753067572911583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,1024,0.011405866344769795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,1536,0.0143477330605189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,128,0.0068896000583966565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,65536,0.379423999786377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,8192,0.04846293528874715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,10240,0.060371200243631996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,12288,0.070796799659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,16384,0.09158506393432617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,7168,0.043287467956542966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,6144,0.037750399112701415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,5120,0.0320906658967336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,4096,0.026012800137201947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,3584,0.023611734310785927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,3072,0.02058239976565043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,2560,0.018308266003926595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,2048,0.015446399648984274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,768,0.008926933010419209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,1024,0.010249599814414978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,512,0.00790293316046397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,1536,0.01297706663608551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,256,0.007095466554164887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,128,0.006097066899140676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,65536,0.3541162808736166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,10240,0.05792533159255982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,8192,0.047169065475463866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,12288,0.06881066958109537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,7168,0.042046932379404704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,16384,0.09003413518269857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,6144,0.03663146495819092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,4096,0.024893865982691447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,5120,0.030686932802200317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,3584,0.022769065697987874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,3072,0.019916800657908122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,2560,0.017803732554117838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,2048,0.015069866180419922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,1024,0.00997866690158844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,512,0.007597866654396057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,768,0.008687999844551087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,1536,0.012724266449610392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,256,0.006779733300209046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,128,0.005898666878541311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,65536,0.33862826029459636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,6144,0.6794335683186848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,7168,0.8035989125569662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,5120,0.5552682876586914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,8192,0.8861098607381186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,10240,1.1120714823404947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,4096,0.4669493357340495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,2560,0.3080287933349609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,3072,0.35828161239624023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,3584,0.42212905883789065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,12288,1.3242603302001954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,2048,0.2649247964223226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,512,0.1216586669286092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,256,0.10667626857757569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,1536,0.21538666089375816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,768,0.15042239824930828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,1024,0.16492692629496258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,128,0.09708267052968343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,16384,1.7879039764404296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,10240,0.2856831868489583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,7168,0.2052512009938558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,12288,0.33779306411743165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,8192,0.22969279289245606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,6144,0.17948373158772785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,16384,0.4391039848327637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,4096,0.12359039783477783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,5120,0.14920639991760254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,2560,0.08282453219095866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,1024,0.048478933175404866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,3584,0.11021866798400878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,768,0.043442134062449136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,3072,0.09720319906870524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,1536,0.06093013286590576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,512,0.036396801471710205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,2048,0.0726090669631958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,128,0.029496532678604127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,256,0.031310933828353885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,8192,0.17562774022420247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,10240,0.21656533877054848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,7168,0.15661439895629883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,12288,0.2597109317779541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,16384,0.34663146336873374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,6144,0.13543999989827474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,5120,0.11316906611124675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,4096,0.09488639831542969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,3584,0.08498133023579915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,2048,0.056542932987213135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,3072,0.0748416026433309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,2560,0.06381013393402099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,768,0.03398186763127645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,1024,0.03768320083618164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,1536,0.04737386703491211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,512,0.028864000240961713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,256,0.024951465924580894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,128,0.023101866245269775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,65536,2.0054752349853517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,12288,0.2196671962738037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,16384,0.291050656636556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,8192,0.1485866705576579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,10240,0.18453332583109539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,7168,0.13933119773864747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,5120,0.09752853711446127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,3584,0.07189546426137289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,3072,0.06458133459091187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,4096,0.07982933521270752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,6144,0.11621332963307698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,65536,1.372988764444987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,2560,0.05549120108286539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,2048,0.04848426580429077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,1536,0.04113173484802246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,1024,0.03281173308690389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,512,0.02577280004819234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,768,0.029395200808842975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,256,0.02141439914703369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,128,0.019454934199651084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,8192,0.12328320344289143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,10240,0.15268160502115885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,7168,0.10847787062327068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,16384,0.23646186192830404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,12288,0.1797877311706543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,3584,0.06000106732050577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,4096,0.06670293013254801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,6144,0.0955018679300944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,65536,1.168551508585612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,3072,0.05360320011774698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,5120,0.08065706888834635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,1536,0.03421760002772013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,256,0.017819732427597046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,2560,0.046054399013519286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,768,0.025176533063252765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,512,0.021284266312917074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,2048,0.040753066539764404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,1024,0.02776106595993042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,128,0.016544000307718913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,10240,0.15347092946370441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,12288,0.18351573944091798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,65536,0.9519903818766275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,8192,0.12432320117950439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,16384,0.24480320612589518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,7168,0.11026240189870198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,6144,0.09323626359303792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,4096,0.06805333296457926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,3072,0.05132266680399576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,3584,0.05814826488494873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,5120,0.07952106793721517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,2560,0.04511786699295044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,1536,0.030508800347646074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,256,0.013963733116785684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,512,0.017195733388264973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,768,0.020498132705688475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,2048,0.037468798955281574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,1024,0.025244800249735515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,65536,0.9648480097452798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,128,0.012506666779518127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,8192,0.09755733013153076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,12288,0.14150400161743165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,10240,0.11835520267486573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,16384,0.1861194610595703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,6144,0.07627840042114258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,7168,0.08544106483459472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,4096,0.05312106609344482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,5120,0.06338346799214681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,1024,0.022612265745798745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,3584,0.04799573421478272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,3072,0.04252586762110393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,1536,0.027475200096766156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,512,0.017485866943995156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,2560,0.03737066586812337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,2048,0.032767999172210696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,128,0.01374613344669342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,768,0.020513067642847695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,256,0.014539733529090881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,12288,0.159662930170695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,10240,0.1343008041381836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,65536,0.7245770772298177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,16384,0.19213973681132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,8192,0.10265599886576335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,7168,0.0863146702448527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,6144,0.07297066847483316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,4096,0.05300586620966593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,3072,0.040240001678466794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,2560,0.036183468500773114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,5120,0.06489813327789307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,3584,0.04835626681645711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,256,0.011122133334477742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,2048,0.030705066521962483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,512,0.01365546683470408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,1536,0.024155733982721965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,768,0.016718933979670204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,65536,0.794757334391276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,1024,0.02039146622021993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,128,0.009969066580136616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,7168,0.06272426843643189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,16384,0.13556267420450846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,8192,0.07078826427459717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,10240,0.08708053429921468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,5120,0.046937600771586103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,12288,0.10511146386464436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,3584,0.035923198858896895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,6144,0.055819733937581384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,2560,0.028142933050791425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,2048,0.02492799957593282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,4096,0.03965866565704346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,3072,0.03182506759961446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,512,0.013498666882514953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,128,0.010760533809661865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,1024,0.017384533087412515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,1536,0.021065600713094077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,768,0.01539306640625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,256,0.011455999811490376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,8192,0.07124693393707275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,7168,0.06323093175888062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,10240,0.0904800017674764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,12288,0.1073525349299113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,65536,0.5122463862101237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,16384,0.13832640647888184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,6144,0.05502293507258097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,5120,0.04673706690470378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,4096,0.03957120180130005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,3584,0.034867199261983235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,3072,0.030691200494766237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,1536,0.018987733125686645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,2560,0.02723520000775655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,2048,0.02326186696688334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,1024,0.015877333283424378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,768,0.013406933347384135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,256,0.009609599908192951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,512,0.011392000317573547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,128,0.008665600419044494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,65536,0.5425055821736653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,7168,0.056236799558003744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,10240,0.07851626873016357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,12288,0.09425493081410727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,8192,0.06469226678212484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,16384,0.12317972977956135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,6144,0.04929173390070597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,3072,0.0278442660967509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,4096,0.03516159852345784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,3584,0.032440533240636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,5120,0.04331626494725545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,2560,0.02547520001729329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,2048,0.02217493255933126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,65536,0.4731146812438965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,256,0.00956053336461385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,512,0.011368532975514729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,1536,0.017755732933680216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,768,0.01295253336429596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,1024,0.01479039986928304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,128,0.00874133308728536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,12288,0.08563520113627116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,10240,0.07212479909261069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,6144,0.04568320115407308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,7168,0.05208319822947184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,8192,0.06084160010019938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,16384,0.11461760203043621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,3072,0.02563733259836833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,4096,0.03245546619097392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,5120,0.039988267421722415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,2560,0.02330453395843506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,3584,0.03032960096995036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,2048,0.0201365331808726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,1536,0.016089600324630738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,65536,0.43728958765665693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,128,0.007947733501593272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,256,0.008789333701133727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,512,0.010373333096504211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,1024,0.013345066706339517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,768,0.011729066570599873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,8192,0.04572906494140625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,6144,0.035651199022928876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,12288,0.0646346648534139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,7168,0.040047999223073324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,16384,0.0859551986058553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,10240,0.05591466824213663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,5120,0.03078826665878296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,3584,0.023735467592875162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,1536,0.013795199990272521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,4096,0.025733333826065064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,1024,0.011505066355069479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,2048,0.016669867436091106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,256,0.008356266220410665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,3072,0.020858667294184365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,2560,0.018717867136001588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,768,0.010834133625030518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,512,0.009706667065620423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,128,0.007828266421953837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,12288,0.05903786818186442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,16384,0.07756053606669108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,10240,0.05135573148727417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,8192,0.042395734786987306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,6144,0.032229334115982056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,65536,0.32012694676717124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,7168,0.03765866756439209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,5120,0.02805546720822652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,4096,0.02363413373629252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,3072,0.01909653345743815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,3584,0.02170026699701945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,2560,0.01725119948387146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,2048,0.015019733707110086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,1536,0.012141866485277812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,512,0.008653866251309712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,128,0.006748799979686737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,1024,0.010290132959683736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,256,0.00740479975938797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,768,0.009578667084376017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,65536,0.30054826736450196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,12288,0.054554665088653566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,16384,0.06971200307210287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,6144,0.029829333225886028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,8192,0.03771626551946004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,10240,0.047443199157714847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,7168,0.03319466710090637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,4096,0.021526400248209634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,5120,0.025931733846664428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,3072,0.016986666123072307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,2048,0.013005866607030233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,3584,0.01920959949493408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,2560,0.015363199512163797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,65536,0.2613365332285563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,256,0.007211733361085255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,768,0.008917333682378133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,128,0.006684799989064534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,512,0.008157866696516674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,1024,0.009705600142478944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,1536,0.01154026687145233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,12288,0.04925653139750163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,10240,0.043619199593861895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,8192,0.034746666749318436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,7168,0.03168533245722453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,6144,0.026953599850336712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,16384,0.0657045324643453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,3072,0.015989333391189575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,4096,0.020039467016855876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,5120,0.0233024001121521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,3584,0.0177130659421285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,2560,0.014200533429781595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,2048,0.012180266777674358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,65536,0.24768640200297037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,128,0.005905066430568695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,256,0.006300800045331319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,768,0.008105599880218506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,1536,0.010454400380452474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,512,0.007342933118343354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,1024,0.008771199981371562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,7168,0.02840106685956319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,6144,0.025690666834513348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,10240,0.04137066602706909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,8192,0.03343360026677449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,12288,0.04712959925333659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,16384,0.05989120006561279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,4096,0.01851093371709188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,3072,0.015364266435305276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,5120,0.021689599752426146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,3584,0.017100799083709716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,2048,0.011594667037328085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,65536,0.22818560600280763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,2560,0.013530666629473368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,256,0.006380799909432728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,512,0.007274666428565979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,1024,0.008317866424719492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,128,0.005825066566467285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,768,0.007845333218574524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,1536,0.010140800476074218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,7168,0.027185066541035967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,8192,0.030372265974680585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,6144,0.024072533845901488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,10240,0.03870933453241984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,12288,0.0446176012357076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,16384,0.057749334971110025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,65536,0.21484799385070802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,4096,0.017461333672205606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,3072,0.01460906664530436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,5120,0.020666666825612388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,3584,0.016578132907549538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,2560,0.013035733501116434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,2048,0.011130666732788086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,128,0.005618133147557576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,1024,0.00804373323917389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,256,0.0060703997810681665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,768,0.007608533402283986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,512,0.007041066884994507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,1536,0.009784533580144247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,6144,0.02379093368848165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,8192,0.030425600210825604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,7168,0.02699626684188843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,10240,0.03747733434041341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,12288,0.04424533446629842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,16384,0.057707734902699793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,5120,0.020679465929667153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,3072,0.014313600460688271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,4096,0.017504000663757326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,2560,0.012973866860071816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,3584,0.016334933042526246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,2048,0.011113599936167399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,65536,0.21448960304260253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,512,0.007047466437021891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,1536,0.009847467144330341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,128,0.0056309332450230915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,1024,0.00804799993832906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,768,0.00761599987745285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,256,0.006129066646099091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,5120,0.5304970741271973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,6144,0.6377514521280925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,7168,0.7328181584676107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,8192,0.8244778951009115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,10240,1.0792063395182292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,12288,1.263326899210612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,4096,0.41823145548502605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,3584,0.37290452321370443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,3072,0.3189258575439453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,2560,0.26905705134073893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,16384,1.7301205952962238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,2048,0.2058506647745768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,1536,0.1663274606068929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,256,0.07780586878458659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,128,0.07603306770324707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,512,0.09269440174102783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,1024,0.12536213397979737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,768,0.11636266708374024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,7168,0.18013866742451984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,10240,0.2705482800801595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,12288,0.3192970593770345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,8192,0.2048415978749593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,6144,0.16027199427286784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,16384,0.41094506581624346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,5120,0.13308160305023192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,2560,0.07230719725290934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,4096,0.10445120334625244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,3584,0.09242880344390869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,3072,0.08076480229695639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,2048,0.0567573348681132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,256,0.024394667148590087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,128,0.022375466426213582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,512,0.02808319926261902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,1536,0.047126400470733645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,768,0.0339466651280721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,1024,0.03779199918111165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,12288,0.22551253636678062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,65536,1.7134133656819661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,10240,0.22014932632446288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,8192,0.15758293469746906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,16384,0.29468905131022133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,7168,0.13501013120015462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,6144,0.1155733346939087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,3072,0.06032853523890177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,4096,0.0800160010655721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,3584,0.07230079968770345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,5120,0.1023146629333496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,2560,0.05432746807734171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,256,0.01606613298257192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,512,0.019332265853881835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,2048,0.04559573332468669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,768,0.02355626622835795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,1536,0.03475199937820435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,1024,0.03054186701774597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,128,0.01397119959195455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,65536,1.238870366414388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,12288,0.1911274592081706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,10240,0.1834272066752116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,8192,0.13721280097961425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,16384,0.25628693898518884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,7168,0.12231040000915527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,6144,0.10452480316162109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,3072,0.05329813162485758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,4096,0.07084906895955403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,2560,0.04781653483708699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,5120,0.08718720277150473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,3584,0.063155198097229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,2048,0.04081813494364421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,1536,0.031703466176986696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,512,0.01837973395983378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,256,0.015525333086649575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,768,0.022219733397165934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,1024,0.026768000920613606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,128,0.013316266735394797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,65536,1.0245087941487632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,10240,0.14406612714131672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,7168,0.09473280111948648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,12288,0.16562560399373372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,8192,0.10553493499755859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,16384,0.21087892850240073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,6144,0.08354880015055338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,5120,0.07036693096160888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,4096,0.0565280000368754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,3584,0.04995413223902385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,2560,0.03891199827194214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,3072,0.04383893410364787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,2048,0.03192533254623413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,1536,0.02667413353919983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,512,0.016387200355529784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,256,0.013705600301424661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,768,0.019372800985972084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,1024,0.021460266908009847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,128,0.013545599579811097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,65536,0.8759936014811197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,7168,0.0845749298731486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,12288,0.14247466723124186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,10240,0.13117012977600098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,8192,0.10315626462300617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,6144,0.07317333221435547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,16384,0.1815989335378011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,3072,0.0392682671546936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,4096,0.04991999864578247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,2560,0.03513706525166829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,2048,0.03025493423144023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,3584,0.046054399013519286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,5120,0.06363626718521118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,1536,0.02323626677195231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,512,0.013533866405487061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,768,0.016033066312472026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,256,0.010874666770299276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,128,0.009852799773216247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,1024,0.01989013353983561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,65536,0.7776021321614583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,6144,0.06783466339111328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,12288,0.14783040682474774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,10240,0.1252127965291341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,7168,0.07568106651306153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,8192,0.08645226955413818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,16384,0.1820064067840576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,4096,0.04727360010147095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,3072,0.035786668459574386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,2560,0.032075732946395874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,5120,0.05570559899012247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,3584,0.04011199871699016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,2048,0.026712532838185626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,1536,0.021449599663416544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,128,0.009884799520174663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,256,0.010889599720637005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,512,0.01286079982916514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,1024,0.018583466609319053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,768,0.015476266543070475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,65536,0.7868832270304362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,6144,0.055958398183186854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,12288,0.1059114694595337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,7168,0.06853013038635254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,10240,0.10173226992289226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,8192,0.07476692994435628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,16384,0.1414026737213135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,3072,0.031188267469406127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,4096,0.039589333534240725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,5120,0.049243732293446855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,2560,0.027749333779017133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,3584,0.03598293463389079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,2048,0.02376213272412618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,65536,0.5466282526652019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,128,0.009027199943860371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,1536,0.0195360004901886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,256,0.009929600358009338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,512,0.011381333072980244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,768,0.014020267128944396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,1024,0.0164192001024882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,7168,0.05755840142567953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,12288,0.10610986550649007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,10240,0.09180906613667807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,6144,0.051702400048573814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,8192,0.06059733231862387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,16384,0.13375040690104167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,3072,0.027127466599146527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,4096,0.03616533279418945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,5120,0.04054720004399617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,3584,0.029582933584849043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,2560,0.024637866020202636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,2048,0.020148267348607383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,65536,0.5520416259765625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,128,0.008220799763997396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,768,0.011819733182589214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,512,0.010221866766611735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,256,0.008985599875450135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,1536,0.016988799969355265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,1024,0.014694399634997048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,6144,0.047945598761240646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,7168,0.051933864752451576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,12288,0.09404266675313314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,8192,0.059630934397379556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,10240,0.08616106510162354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,16384,0.12937599817911785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,3072,0.025410133600234985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,4096,0.032059733072916666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,3584,0.029241599639256793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,2560,0.023242666323979696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,5120,0.0406058669090271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,2048,0.02002240022023519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,65536,0.5249439875284831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,128,0.008012799918651581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,256,0.008690133690834045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,512,0.010103467106819152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,768,0.011747200290362041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,1536,0.016634666919708253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,1024,0.013458133737246195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,6144,0.046669868628184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,10240,0.0801098664601644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,12288,0.09421333471934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,7168,0.0518175999323527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,8192,0.056383999188741055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,16384,0.12236053148905437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,3072,0.023924267292022704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,2560,0.021612799167633055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,4096,0.030898133913675945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,3584,0.027489066123962402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,5120,0.037374933560689286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,2048,0.01832746664683024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,65536,0.5025823911031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,128,0.007593599955240886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,1536,0.014784000317255654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,512,0.00944640040397644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,768,0.010898133118947346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,1024,0.012643200159072877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,256,0.008266666531562805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,6144,0.03768853346506755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,12288,0.07319680054982504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,7168,0.04445120096206665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,10240,0.0631722648938497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,8192,0.049322664737701416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,16384,0.09105707009633382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,3072,0.021411200364430748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,4096,0.02831466595331828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,3584,0.023081600666046143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,65536,0.35364160537719724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,2560,0.019126399358113607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,5120,0.031133866310119628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,2048,0.015513599912325541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,128,0.006712533533573151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,256,0.007336533566315968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,512,0.008183466891447704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,1536,0.013101866841316223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,768,0.009451733032862345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,1024,0.010778666536013285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,12288,0.06772159735361735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,10240,0.05708160003026327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,7168,0.041587201754252116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,16384,0.08927679856618245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,6144,0.035021865367889406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,8192,0.045788800716400145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,65536,0.3535434722900391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,5120,0.030297599236170453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,3072,0.01946773330370585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,4096,0.02609279950459798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,2560,0.017398399114608765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,3584,0.02176213264465332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,2048,0.01455893317858378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,1536,0.012077866991360982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,256,0.0067562664548556015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,128,0.006236800054709116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,512,0.007693866888682048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,768,0.00853760043780009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,1024,0.009799466530481974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,16384,0.07957119941711426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,12288,0.05963413317998251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,8192,0.038970665136973066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,10240,0.04985386530558268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,6144,0.03113600015640259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,7168,0.034355199337005614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,65536,0.3167914708455404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,3072,0.016645333170890807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,3584,0.018377600113550822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,2560,0.014806399742762247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,4096,0.021142399311065672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,5120,0.0254805326461792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,2048,0.012430933117866517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,1536,0.010580266515413921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,128,0.005958400170008342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,256,0.006439466774463653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,512,0.007354666789372762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,768,0.008250666658083598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,1024,0.009170132875442504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,12288,0.05120533307393392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,6144,0.025382399559020996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,16384,0.06433066527048746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,7168,0.028492800394694012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,10240,0.04482560157775879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,8192,0.03257173299789429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,65536,0.27703892389933266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,3072,0.014707199732462563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,4096,0.018114133675893148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,5120,0.02151040037473043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,2560,0.013208533326784769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,2048,0.010843732953071594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,3584,0.015863466262817382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,1536,0.009267200032869976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,768,0.007204266885916392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,512,0.0062730665008227035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,1024,0.008165333171685536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,256,0.005760000149408976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,128,0.005346133311589559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,6144,0.02301759918530782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,12288,0.043828264872233076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,8192,0.029917865991592407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,16384,0.05663893222808838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,10240,0.03623466491699219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,7168,0.02635200023651123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,65536,0.2601642608642578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,4096,0.016667733589808144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,3072,0.013486933708190919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,2560,0.012216533223787945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,5120,0.02031146685282389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,3584,0.015226667126019796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,2048,0.010547199845314026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,1536,0.009125333031018574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,128,0.005369600156943003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,512,0.0062943999965985615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,256,0.00573333352804184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,1024,0.00804799993832906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,768,0.007095466554164887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,16384,0.053783468405405675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,12288,0.042520534992218015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,8192,0.02797973354657491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,10240,0.03473493258158366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,6144,0.021964800357818604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,7168,0.024715733528137208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,65536,0.23223360379536948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,5120,0.01882560054461161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,3072,0.012894933422406515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,2048,0.010016000270843506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,4096,0.01569706698258718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,3584,0.014250666896502177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,2560,0.011737599968910217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,1536,0.008694400389989216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,768,0.0069567998250325514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,256,0.005598933498064677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,1024,0.00782400021950404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,512,0.006179200112819671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,128,0.005202133456865946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,16384,0.046147199471791585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,6144,0.019697066148122153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,7168,0.021756800015767415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,10240,0.030739200115203858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,12288,0.035819733142852785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,8192,0.024307199319203696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,65536,0.17689065933227538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,3072,0.012171733379364013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,4096,0.014538666605949402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,2048,0.009268266956011455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,5120,0.0168287992477417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,3584,0.013267200191815695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,2560,0.01104960044225057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,1536,0.008501332998275758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,128,0.00488319993019104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,256,0.005264000097910563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,768,0.006551466882228851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,1024,0.007239466905593872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,512,0.005835733314355215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,16384,0.04578773180643718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,7168,0.021676800648371377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,8192,0.02412160038948059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,12288,0.03447573184967041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,10240,0.029625600576400755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,6144,0.018870399395624796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,65536,0.17054400444030762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,5120,0.01667520006497701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,4096,0.01406719982624054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,3072,0.011724799871444702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,2048,0.009065600236256917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,3584,0.013165866335233053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,2560,0.010875733693440755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,1536,0.00844373305638631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,512,0.0057429333527882894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,768,0.006485333542029063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,256,0.005222400029500326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,128,0.004913066824277243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,1024,0.007236266632874806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,5120,0.3462901433308919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,6144,0.41544214884440106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,7168,0.479751459757487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,8192,0.5403167724609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,4096,0.276584529876709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,10240,0.7159274419148762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,2560,0.1803637345631917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,12288,0.8411146799723307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,3584,0.24515840212504067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,3072,0.2135647932688395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,2048,0.1406559944152832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,1024,0.08596906661987305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,1536,0.116046937306722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,256,0.05461333195368449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,768,0.0803551991780599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,512,0.06390613317489624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,16384,1.0668020884195963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,128,0.052042667071024576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,10240,0.18070400555928548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,8192,0.13389120101928711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,7168,0.11862719853719075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,12288,0.21332693099975586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,6144,0.1051957368850708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,16384,0.2692810694376627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,5120,0.08830186525980631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,4096,0.06950293382008871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,3584,0.06286400159200033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,2560,0.04803413152694702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,3072,0.05480426549911499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,2048,0.03991573254267375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,1536,0.03369706471761068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,1024,0.026658133665720625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,768,0.02390186587969462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,512,0.02026559909184774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,256,0.01768853267033895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,128,0.01588266690572103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,7168,0.09034773508707682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,10240,0.1398848056793213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,12288,0.15763840675354004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,8192,0.10247680346171062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,16384,0.20544853210449218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,5120,0.06650559902191162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,65536,1.1370773315429688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,6144,0.07973972956339517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,4096,0.0535701314608256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,3584,0.048495999972025555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,3072,0.04137279987335205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,2560,0.03707520167032878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,2048,0.032891732454299924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,1536,0.026386133829752606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,768,0.019058134158452353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,512,0.016300800442695617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,1024,0.0210698664188385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,256,0.013931733369827271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,128,0.013079466422398886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,65536,0.8430271784464518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,12288,0.1315658648808797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,10240,0.1318303982416789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,16384,0.17185600598653156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,7168,0.0809269348780314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,6144,0.07141013145446777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,8192,0.09032533168792725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,3072,0.03726826508839925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,5120,0.0587936004002889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,2560,0.03364479939142863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,4096,0.04801173210144043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,3584,0.04433173338572184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,2048,0.027856000264485675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,1536,0.023041067520777385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,256,0.011562666296958924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,512,0.014120533068974813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,128,0.010539733370145162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,768,0.016196266810099284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,1024,0.019782400131225585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,65536,0.7258037567138672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,8192,0.06785706679026285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,7168,0.061434666315714516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,10240,0.0970858653386434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,12288,0.10965332984924317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,16384,0.1405194600423177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,6144,0.0556277314821879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,3584,0.034142935276031496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,3072,0.0295413335164388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,4096,0.0382207989692688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,5120,0.04729919830958049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,2560,0.02619733413060506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,2048,0.022632533311843873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,512,0.012523733576138816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,1536,0.019511467218399046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,1024,0.016008533040682473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,768,0.014506666858990987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,128,0.010222933689753215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,256,0.010466133554776508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,12288,0.09263359705607097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,8192,0.06908693313598632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,65536,0.5832661310831706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,10240,0.07974080244700113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,7168,0.05666133165359497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,16384,0.12301759719848633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,6144,0.04910186529159546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,5120,0.04464426835378011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,4096,0.03589119911193848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,3584,0.03264426589012146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,2560,0.024682666858037314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,3072,0.026929066578547163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,1536,0.017374932765960693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,2048,0.021625600258509316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,1024,0.014913066228230795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,768,0.012427733341852824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,512,0.010341333349545796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,256,0.008725333213806152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,128,0.007941333452860515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,10240,0.0856117328008016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,65536,0.48400106430053713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,7168,0.05065066814422607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,12288,0.09719253381093343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,8192,0.05778773228327433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,16384,0.11971626281738282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,5120,0.038007465998331706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,6144,0.04705706834793091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,3584,0.02874026695887248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,4096,0.03153706590334575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,3072,0.024690133333206177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,2560,0.022292266289393105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,2048,0.01925119956334432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,512,0.009823999802271525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,128,0.008041599889596303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,768,0.011582932869593303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,1024,0.013929599523544311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,1536,0.016170666615168253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,256,0.008634666601816814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,65536,0.5106538772583008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,7168,0.04527466694513957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,12288,0.08641599814097087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,10240,0.07001279989878337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,16384,0.1136799971262614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,8192,0.05240533351898193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,6144,0.044660266240437826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,4096,0.028568534056345622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,3072,0.02224000096321106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,5120,0.034587732950846356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,3584,0.02658560077349345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,2560,0.020466132958730062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,65536,0.4892896016438802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,2048,0.017544533809026083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,1536,0.014682666460673014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,512,0.008845866719881693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,256,0.00780266672372818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,768,0.01016960044701894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,1024,0.012449066837628682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,128,0.007293866574764251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,7168,0.03455679814020793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,10240,0.052300798892974856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,12288,0.060813868045806886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,8192,0.038083199659983316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,16384,0.07598933378855387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,6144,0.03131200075149536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,5120,0.02595413327217102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,4096,0.02191466689109802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,2560,0.016382933656374613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,3584,0.020179200172424316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,3072,0.01789439916610718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,65536,0.3110901196797689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,2048,0.01434879998366038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,1536,0.012732799847920737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,256,0.007518933216730754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,512,0.008294400076071422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,128,0.007229866584142049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,768,0.009199999769528707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,1024,0.010580266515413921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,12288,0.053133865197499595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,7168,0.033139199018478394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,10240,0.04518826802571614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,16384,0.06940906842549642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,6144,0.028857600688934327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,8192,0.03856639862060547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,65536,0.26952107747395837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,3072,0.017036799589792886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,4096,0.02109866738319397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,2560,0.015460266669591268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,5120,0.02621440092722575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,3584,0.020268799861272176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,2048,0.013859200477600097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,1536,0.011852799852689107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,256,0.0065994665026664736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,128,0.006242133180300395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,512,0.007435733576615651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,768,0.008227199812730153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,1024,0.009928533434867859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,6144,0.027266132831573486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,10240,0.04103039900461833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,12288,0.05307733217875162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,7168,0.02991360028584798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,16384,0.06482453346252441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,8192,0.03419093290964763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,65536,0.2678741455078125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,3072,0.015887999534606935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,4096,0.01957013408342997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,3584,0.018295466899871826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,5120,0.02324906587600708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,2048,0.0127018670241038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,2560,0.014631467064221701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,1536,0.010956799983978272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,256,0.00652266691128413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,512,0.0072736000021298725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,768,0.007834666470686594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,128,0.006126933296521505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,1024,0.009037866195042928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,6144,0.024884267648061117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,12288,0.04956373373667399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,10240,0.04190613428751628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,16384,0.06045013268788656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,7168,0.02733866572380066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,8192,0.03171306649843852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,65536,0.2685311953226725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,4096,0.017761067549387614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,3072,0.014520532886187234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,5120,0.02150719960530599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,2560,0.013401599725087485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,2048,0.011716266473134358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,3584,0.0171509325504303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,1536,0.009924266735712688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,512,0.0068234667181968685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,128,0.005717333157857259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,256,0.006131199995676676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,1024,0.008382933338483174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,768,0.007369600236415863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,16384,0.060946134726206455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,10240,0.040573867162068684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,12288,0.04638826847076416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,7168,0.02738773425420125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,8192,0.031192533175150555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,6144,0.024860799312591553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,65536,0.24374186197916664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,5120,0.020755199591318767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,4096,0.01803413430849711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,3072,0.014510933558146158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,3584,0.016702934106191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,2048,0.010843732953071594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,2560,0.012714667121569314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,1536,0.009716266393661499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,1024,0.008122666676839193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,256,0.006312533219655354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,128,0.0059232001503308615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,512,0.0068906664848327635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,768,0.0072618668278058365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,12288,0.03869119882583618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,6144,0.020965333779652914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,7168,0.02342826724052429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,10240,0.033427198727925614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,16384,0.049658668041229245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,8192,0.026639999945958455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,65536,0.21626879374186198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,4096,0.015530666708946228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,3072,0.012750933567682901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,5120,0.018730666240056357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,3584,0.014769066373507181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,2560,0.011109333237012227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,2048,0.00972266693909963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,1536,0.008649599552154542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,128,0.00528959979613622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,512,0.006142933170000712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,256,0.005569066603978475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,768,0.006663466493288676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,1024,0.00761599987745285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,16384,0.042444801330566405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,8192,0.02275200088818868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,10240,0.03419839938481649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,6144,0.019296000401178993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,7168,0.021269333362579346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,12288,0.03878186543782552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,65536,0.19514452616373698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,5120,0.01623146633307139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,2048,0.008808533350626629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,4096,0.013581867019335428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,3072,0.01111253301302592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,2560,0.010483200351397198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,3584,0.013301333785057068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,1536,0.008389332890510559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,1024,0.007080533107121785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,256,0.005542399982611338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,128,0.0052138666311899815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,512,0.00602346658706665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,768,0.0063967997829119366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,16384,0.043910400072733564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,6144,0.019035732746124266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,12288,0.03332800070444743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,7168,0.021669334173202513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,8192,0.0240831991036733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,10240,0.029285333553949994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,65536,0.18244585990905762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,3072,0.011026133100191753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,4096,0.013828266660372415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,2560,0.010285866260528565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,5120,0.016479999820391337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,2048,0.009074133634567261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,3584,0.013190399607022604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,1536,0.008164266745249431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,128,0.005132799843947092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,512,0.005996799965699514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,256,0.005434666574001312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,768,0.006437333424886067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,1024,0.0073088000218073535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,16384,0.038915201028188066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,7168,0.01895466645558675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,6144,0.016706132888793947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,10240,0.02656746705373128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,12288,0.029674667119979858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,8192,0.021102933088938396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,65536,0.15223573048909506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,5120,0.014865066607793173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,3072,0.010610133409500122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,4096,0.012458667159080505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,2560,0.009976533055305482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,3584,0.012351999680201214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,2048,0.008425600330034892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,1536,0.008187733093897502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,1024,0.006982400019963582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,128,0.005027199784914652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,768,0.006202666461467743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,256,0.005238399902979533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,512,0.005806933343410492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,16384,0.038024532794952395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,12288,0.02834026614824931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,7168,0.018385066588719686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,10240,0.02507946689923604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,8192,0.020206934213638304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,6144,0.01627946694691976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,65536,0.14927466710408527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,4096,0.012171733379364013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,5120,0.014297599593798319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,3584,0.012156800429026286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,3072,0.0102783997853597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,2560,0.009700266520182292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,2048,0.008350933591524761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,1536,0.007898666461308797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,1024,0.006772266825040181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,768,0.006000000238418579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,128,0.004782933493455251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,512,0.005691733459631602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,256,0.005096533397833506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,12288,0.028283733129501342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,10240,0.024065067370732628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,8192,0.0200981338818868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,16384,0.03664746681849162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,6144,0.016149333119392394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,7168,0.018285866578420004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,5120,0.014242133498191834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,3584,0.011907200018564861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,65536,0.13712639808654786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,4096,0.012062933047612507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,3072,0.010083199540774027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,2560,0.009531733393669129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,2048,0.008137600123882293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,768,0.005905066430568695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,1024,0.006738133231798808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,1536,0.007915733257929485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,512,0.005554133156935374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,256,0.005063466727733612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,128,0.004810666541258494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,5120,0.2546208063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,6144,0.3093824068705241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,7168,0.34859838485717776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,8192,0.39570239384969075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,10240,0.5311168034871419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,12288,0.6199978510538737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,4096,0.1997375965118408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,2560,0.1352394739786784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,3584,0.17611093521118165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,3072,0.1543285369873047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,768,0.06298453410466512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,1536,0.0921621322631836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,256,0.04287466605504354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,512,0.04962773323059082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,1024,0.06721173127492269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,16384,0.7983840306599934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,128,0.0404586672782898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,2048,0.11085013548533122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,8192,0.10277120272318523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,10240,0.1297919988632202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,7168,0.08812586466471353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,12288,0.15153600374857584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,4096,0.05252266724904379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,16384,0.19724480311075848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,5120,0.06577599843343099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,3584,0.04736959934234619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,6144,0.07905813058217367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,1024,0.02120959957440694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,3072,0.04120959838231404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,2560,0.03639466762542724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,1536,0.027687466144561766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,2048,0.0325162669022878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,512,0.016523733735084534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,128,0.01300373375415802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,256,0.014375467101732889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,768,0.019242666165033975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,12288,0.13118080298105878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,16384,0.16147732734680176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,8192,0.0830847978591919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,10240,0.10950613021850586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,7168,0.07343786557515462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,65536,0.8332991917928061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,6144,0.06389333407084147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,4096,0.042499200503031416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,3584,0.03883306582768758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,5120,0.053523198763529456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,2560,0.0316650668780009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,3072,0.03400853474934896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,2048,0.02642773389816284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,1536,0.021065600713094077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,768,0.014824533462524414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,512,0.013009066383043924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,1024,0.017723733186721803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,256,0.010714667042096455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,65536,0.7125333150227864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,128,0.009750399986902874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,10240,0.08402346769968669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,8192,0.07275733153025309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,12288,0.1
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,16384,0.13075839678446452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,7168,0.0624672015508016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,6144,0.05221653381983439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,5120,0.04599253336588542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,3072,0.028777599334716797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,2560,0.02632853388786316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,2048,0.023167999585469563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,4096,0.03768746852874756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,3584,0.03436479965845744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,1536,0.01877973278363546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,1024,0.015758933623631795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,512,0.011901866396268208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,768,0.01384213368097941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,256,0.009733333190282186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,128,0.008920533458391826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,65536,0.5047722816467285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,7168,0.05008426507314047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,12288,0.09146773020426432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,10240,0.0709333340326945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,8192,0.05783040126164755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,16384,0.11575040022532146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,6144,0.045628801981608076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,3584,0.027910399436950683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,3072,0.02472213308016459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,4096,0.031385600566864014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,5120,0.03732906579971314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,2560,0.022201599677403767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,65536,0.49364051818847654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,512,0.0100000003973643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,768,0.011963733037312825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,256,0.008898133039474487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,2048,0.01956160068511963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,1536,0.016215466459592185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,1024,0.013767466942469279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,128,0.008191999793052674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,12288,0.07727893193562826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,7168,0.04794986645380656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,10240,0.0664576013882955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,8192,0.055054934819539394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,16384,0.10423786640167236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,6144,0.0404639999071757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,3072,0.02360960046450297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,2560,0.02146240075429281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,4096,0.029674667119979858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,3584,0.027194666862487792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,2048,0.019099734226862588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,5120,0.036187732219696046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,65536,0.46456000010172527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,1536,0.015965867042541503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,256,0.008496000369389852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,128,0.007923200229803721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,768,0.011660800377527872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,512,0.009834667046864826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,1024,0.013458133737246195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,6144,0.03954773346583049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,12288,0.07407466570536295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,7168,0.04328320026397705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,10240,0.0677781343460083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,8192,0.05035306612650553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,16384,0.10287893613179525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,5120,0.032790400584538776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,3072,0.021186133225758873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,4096,0.02681279977162679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,2560,0.01948266625404358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,3584,0.02461013396581014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,2048,0.01763199965159098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,65536,0.42888959248860675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,768,0.01036906639734904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,1536,0.014300800363222756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,256,0.007950933277606964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,512,0.009211732943852743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,128,0.007507200042406718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,1024,0.012396799524625142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,7168,0.03364373445510864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,16384,0.07054293155670166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,6144,0.02940906683603923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,8192,0.039892268180847165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,10240,0.04626773198445638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,12288,0.0541919986406962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,2560,0.016085333625475564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,5120,0.026959999402364092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,4096,0.021597866217295328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,3584,0.02055466572443644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,2048,0.014830933014551798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,3072,0.017848533391952515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,1536,0.012376532951990763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,1024,0.010334933797518413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,768,0.008974933624267578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,512,0.007922133306662242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,256,0.007075199981530507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,128,0.006579199930032094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,65536,0.27647040685017904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,7168,0.028699733813603717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,8192,0.032604799667994185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,10240,0.04273706674575806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,12288,0.04838080008824666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,16384,0.061451733112335205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,5120,0.022385066747665404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,6144,0.02646399935086568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,4096,0.01949119965235392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,1024,0.008732799688975017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,3072,0.015655466914176942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,3584,0.017191465695699057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,2560,0.01418239971001943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,1536,0.0108842670917511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,2048,0.013030399878819784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,768,0.007980800171693166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,512,0.007330133517583211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,256,0.006682666639486949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,128,0.006161066889762879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,65536,0.2664405345916748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,16384,0.05823359886805216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,12288,0.043798398971557614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,8192,0.03173866669336955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,6144,0.024371200799942018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,7168,0.027507199843724565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,10240,0.0365664005279541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,2048,0.012758400042851767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,2560,0.013933866222699483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,3072,0.015118933717409768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,4096,0.018125865856806436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,3584,0.01709866722424825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,5120,0.02206933299700419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,65536,0.24584213892618814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,1536,0.010948266585667927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,1024,0.008635733524958293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,768,0.008088533580303193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,512,0.007320533196131389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,256,0.006506666541099548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,128,0.006033066908518473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,16384,0.05402026573816935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,8192,0.029069866736729937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,10240,0.038770135243733725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,7168,0.025310933589935303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,6144,0.022957867383956908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,12288,0.04113920132319133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,2560,0.01293333371480306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,5120,0.020424532890319824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,4096,0.01720959941546122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,3584,0.015811199943224587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,65536,0.2343712011973063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,3072,0.014113066593805948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,2048,0.012036266922950744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,1536,0.009834667046864826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,1024,0.00816319982210795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,768,0.007714133461316426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,512,0.006968533496061961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,256,0.006196266909440359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,128,0.0058218667904535925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,12288,0.04098026752471924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,16384,0.05328533252080282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,8192,0.027708800633748372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,6144,0.02217280069986979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,10240,0.03629120190938313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,7168,0.025276799996693928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,65536,0.21922027269999184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,5120,0.01955946683883667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,3072,0.013713066776593527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,4096,0.016669867436091106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,2048,0.010447999835014344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,2560,0.012680533528327941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,3584,0.014881066481272378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,1024,0.007567999760309856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,256,0.006239999830722809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,128,0.005886933207511902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,512,0.0068341334660847975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,768,0.007256533205509186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,1536,0.009309867024421692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,12288,0.03722453514734904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,6144,0.02026240030924479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,16384,0.04605120023091634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,10240,0.032096000512441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,7168,0.021980800231297812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,8192,0.025443200270334882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,65536,0.19989013671875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,3072,0.012716799974441528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,2560,0.010843732953071594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,4096,0.014900267124176025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,3584,0.013858133554458618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,2048,0.009741866588592529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,5120,0.017811199029286705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,1536,0.008258133133252462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,256,0.005515733361244201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,128,0.00526506652434667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,512,0.006065066655476888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,1024,0.007225599884986877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,768,0.0065184002121289565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,16384,0.043892268339792886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,12288,0.03351999918619792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,6144,0.01893226703008016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,7168,0.02137493292490641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,10240,0.028307199478149414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,8192,0.02447893420855204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,65536,0.16957119305928547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,5120,0.017218132813771568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,4096,0.014125866691271463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,3072,0.011618133385976155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,2560,0.010312533378601075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,3584,0.013555199901262919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,2048,0.009567999839782714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,1536,0.008242133259773254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,1024,0.007147733370463054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,512,0.006102400024731954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,768,0.006554666658242543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,256,0.005530666808287303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,128,0.005198933184146881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,16384,0.040490667025248214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,12288,0.030795733133951824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,6144,0.01804693341255188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,8192,0.02216426730155945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,7168,0.019857066869735717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,10240,0.026925865809122724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,65536,0.16257707277933758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,5120,0.015963733196258545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,3072,0.010886399944623312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,4096,0.013160533706347146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,3584,0.012313600381215413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,2048,0.009100799759229023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,2560,0.009773866335550944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,1536,0.007774933179219563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,1024,0.006862933437029521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,128,0.005160533388455709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,768,0.006331733365853627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,256,0.005388799806435903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,512,0.005858133236567179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,16384,0.040277334054311116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,12288,0.03031040032704671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,4096,0.01250879963239034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,8192,0.0222378671169281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,10240,0.02569813330968221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,6144,0.017451733350753784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,7168,0.019461333751678467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,65536,0.14778560002644855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,5120,0.015561599532763162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,3072,0.01043839951356252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,3584,0.011993599931399028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,2560,0.00953493316968282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,1536,0.0075989335775375364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,1024,0.006621866424878438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,2048,0.008830933769543966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,768,0.006208000083764395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,512,0.005756799876689911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,256,0.0052714665730794275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,128,0.004939733445644379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,12288,0.026360533634821576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,10240,0.02287786602973938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,8192,0.019139200448989868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,16384,0.03348160187403361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,7168,0.016884267330169678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,3584,0.011080533266067505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,3072,0.009797333677609762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,65536,0.1304106632868449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,6144,0.015072000026702882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,2560,0.009230933586756388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,4096,0.011397332946459452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,5120,0.013364266355832419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,512,0.005611733098824819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,2048,0.008224000036716462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,256,0.005150933563709259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,768,0.005946666498978933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,1024,0.00631039987007777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,1536,0.0076117331782976794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,128,0.004839466512203216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,7168,0.016452266772588094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,10240,0.021798400084177653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,16384,0.03353600104649861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,12288,0.02561066746711731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,8192,0.018336000045140584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,4096,0.011228799819946289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,6144,0.01490133305390676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,5120,0.013198933005332947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,2048,0.008096000055472057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,1536,0.0074442664782206226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,65536,0.12414506276448567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,3584,0.010981333255767823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,2560,0.009055999914805095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,3072,0.009742933511734008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,1024,0.006289066871007283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,768,0.005850666761398315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,512,0.005568000177542368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,256,0.005042133231957754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,128,0.00481386681397756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,12288,0.025243733326594037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,16384,0.032585599025090534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,6144,0.014879999558130899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,7168,0.016410666704177856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,10240,0.021793067455291748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,8192,0.018262400229771932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,65536,0.12135573228200276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,5120,0.01316480040550232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,4096,0.011154133081436157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,3584,0.010818133751551311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,3072,0.009589333335558574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,1024,0.006235733131567637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,2560,0.008899199962615966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,2048,0.007963733375072479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,1536,0.007518933216730754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,512,0.005420800050099691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,768,0.0058773333827654515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,256,0.004977066814899444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,128,0.0047199999292691554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,5120,0.17317546208699544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,6144,0.2031775951385498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,7168,0.2325493335723877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,8192,0.258135461807251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,10240,0.3495978673299154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,12288,0.41853866577148435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,4096,0.14293759663899738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,3584,0.12562560240427653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,3072,0.10879360040028889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,2560,0.09261546929677328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,2048,0.08194026947021485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,1536,0.06887253125508627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,512,0.03620800177256266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,128,0.02900159955024719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,768,0.04654826720555623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,16384,0.5413685480753581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,1024,0.050010665257771814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,256,0.03107200066248576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,6144,0.054922668139139805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,7168,0.06375679969787598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,8192,0.07640853722890219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,10240,0.09170880317687988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,12288,0.10536106427510579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,16384,0.13138559659322102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,3584,0.03638293345769246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,5120,0.04705173174540202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,2560,0.028243199984232588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,4096,0.03988800048828125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,3072,0.031947733958562215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,2048,0.02534293333689372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,1536,0.021265067656834922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,1024,0.016377600034077962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,768,0.015054933230082192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,512,0.013032533725102744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,256,0.011001599828402202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,128,0.010216533144315084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,12288,0.08930666446685791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,8192,0.05784746805826822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,10240,0.0762997309366862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,7168,0.051495468616485594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,16384,0.1106005350748698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,4096,0.03212266763051351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,3584,0.029024000962575274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,6144,0.044341333707173664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,5120,0.038753068447113036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,3072,0.025757867097854614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,65536,0.570187759399414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,2560,0.023752532402674355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,768,0.012058666348457337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,1024,0.014020267128944396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,1536,0.016102400422096253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,2048,0.020129066705703736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,256,0.008552533388137818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,128,0.007971199850241344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,512,0.010549333691596986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,65536,0.48275413513183596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,12288,0.08069653511047363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,10240,0.06492799917856852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,16384,0.09924906889597575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,7168,0.04592853387196859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,8192,0.051363201936086025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,6144,0.04258133172988891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,4096,0.02873706618944804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,5120,0.034585599104563394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,3072,0.023178666830062866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,3584,0.026306132475535076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,2560,0.02121386726697286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,2048,0.018204800287882485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,1536,0.014410666624704995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,768,0.010918399691581726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,256,0.007681066791216533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,1024,0.012795733412106833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,512,0.00897279977798462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,65536,0.434768009185791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,128,0.00717439999183019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,8192,0.038482133547465006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,12288,0.0559445341428121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,10240,0.047948801517486574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,7168,0.037870931625366214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,16384,0.07163946628570557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,6144,0.03154453237851461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,5120,0.02700693408648173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,2048,0.015408000349998474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,3584,0.0212501327196757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,3072,0.019397334257761637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,2560,0.017118932803471883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,4096,0.022999467452367146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,1536,0.012947199741999307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,256,0.0074325333038965866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,768,0.009553066889444987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,512,0.008175999919573466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,1024,0.01134399970372518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,128,0.0071936001380284624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,12288,0.05074986616770426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,65536,0.2906463940938314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,7168,0.03236906727155049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,8192,0.036433064937591554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,16384,0.06534293492635092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,10240,0.04311573505401611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,2560,0.015554133057594299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,5120,0.02639893293380737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,4096,0.02121493419011434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,3584,0.019453867276509603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,3072,0.017147733767827352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,1536,0.011660800377527872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,6144,0.028233599662780762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,2048,0.014193066954612732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,128,0.006200533111890157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,1024,0.009801600376764934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,768,0.008226133386294047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,512,0.007211733361085255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,256,0.006538666784763336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,7168,0.02982719937960307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,65536,0.2530346711476644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,8192,0.03252266645431519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,16384,0.0587936004002889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,10240,0.039686401685078934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,12288,0.04702826738357544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,3072,0.01607039968172709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,5120,0.024238934119542442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,3584,0.017434666554133095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,6144,0.025865600506464644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,768,0.00775786687930425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,1536,0.01095466713110606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,4096,0.019313067197799683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,1024,0.009101866682370504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,2560,0.014363732933998109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,256,0.0065290664633115125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,2048,0.013090133666992188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,512,0.0070613334576288865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,128,0.006073600053787232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,16384,0.05349973440170288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,65536,0.24884160359700522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,10240,0.03596693277359009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,5120,0.020198400815327963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,12288,0.041870931784311935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,8192,0.029686399300893146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,7168,0.026681600014368693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,6144,0.02336853345235189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,2048,0.011692800124486287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,3584,0.015715199708938598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,4096,0.01763946612675985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,3072,0.014251733819643656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,2560,0.01316266655921936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,768,0.0072629332542419435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,1536,0.010009599725405376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,256,0.006026666859785716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,1024,0.008409600456555684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,512,0.006651733318964641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,65536,0.22779946327209472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,128,0.005627733469009399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,12288,0.039290666580200195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,16384,0.05025706688563029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,10240,0.0339850664138794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,8192,0.026514132817586262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,7168,0.024366933107376098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,6144,0.02139520049095154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,5120,0.018901334206263224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,4096,0.016065067052841185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,2048,0.011166933178901672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,3072,0.01374399960041046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,65536,0.20759147008260093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,2560,0.012749866644541422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,3584,0.014677332838376364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,256,0.006206933160622915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,1024,0.007540266712506611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,512,0.006648533542950948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,128,0.00591893345117569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,768,0.0071168000499407455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,1536,0.009622400005658466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,6144,0.019687465826670327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,12288,0.03578773339589437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,7168,0.022813866535822548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,8192,0.025432533025741576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,10240,0.030502400795618695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,16384,0.04589759906133016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,65536,0.19205439885457357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,3072,0.012689066926638284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,4096,0.014831999937693277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,3584,0.0139082670211792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,2560,0.011896533767382304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,2048,0.010690133770306904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,5120,0.017407999436060587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,1536,0.00853653351465861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,768,0.006795733173688252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,256,0.005838933090368906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,128,0.005492266515890757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,512,0.006277333199977875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,1024,0.007445333401362102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,16384,0.04326933224995931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,6144,0.018304000298182167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,10240,0.028812799851099653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,12288,0.03361920118331909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,8192,0.024197334051132204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,7168,0.02144213318824768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,65536,0.18135573069254557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,5120,0.016591999928156534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,3072,0.012083199620246888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,4096,0.013967999815940857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,3584,0.013380266229311624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,2560,0.01123199959595998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,2048,0.00999786655108134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,1536,0.007971199850241344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,128,0.005268266797065735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,1024,0.006900266806284587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,256,0.005541333556175232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,768,0.006372266511122386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,512,0.005895466605822245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,16384,0.041317331790924075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,12288,0.03191893299420674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,7168,0.020547199249267577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,10240,0.02687679926554362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,8192,0.02299413283665975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,6144,0.017654399077097573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,65536,0.162883202234904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,5120,0.016123732924461363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,3072,0.011750400066375732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,2560,0.01095466713110606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,4096,0.01362986663977305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,2048,0.009768533706665038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,3584,0.012987732887268066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,1536,0.007740800082683563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,1024,0.0069461335738499955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,256,0.005526400109132131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,128,0.005321600039800008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,512,0.005994666616121928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,768,0.0064064001043637585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,16384,0.03487679958343506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,12288,0.029131732384363812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,7168,0.018964266777038573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,10240,0.026246400674184163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,8192,0.0193066676457723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,6144,0.015929599603017174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,65536,0.1535626729329427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,5120,0.014315733313560485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,4096,0.012264532844225566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,3072,0.010733866691589355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,3584,0.012014933427174886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,2560,0.010061867038408915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,2048,0.008327466746171314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,1536,0.007621333499749501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,1024,0.006570666531721751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,768,0.00621013343334198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,128,0.0051818668842315676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,512,0.0058559998869895935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,256,0.0053941334287325535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,16384,0.03729493220647176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,12288,0.028455466032028198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,10240,0.024154667059580484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,6144,0.01651946703592936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,7168,0.01875093380610148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,8192,0.020526933670043945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,65536,0.1406069278717041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,5120,0.014973866939544677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,4096,0.01223360002040863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,3072,0.010281599561373393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,3584,0.01162986656030019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,2560,0.009498666723569233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,1536,0.007216000060240428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,2048,0.008612266182899475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,1024,0.0066453332702318835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,768,0.006177066763242086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,512,0.005765333275000254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,256,0.005320533116658529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,128,0.0051146666208903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,16384,0.030834132432937623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,12288,0.024307199319203696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,10240,0.02151040037473043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,8192,0.017734400431315103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,6144,0.014570666352907815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,7168,0.016495999693870545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,65536,0.11485866705576579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,5120,0.012582400441169738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,4096,0.010660266876220703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,3072,0.009667199850082398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,3584,0.010629333058993022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,2048,0.007969066500663757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,2560,0.009149866302808125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,1536,0.0073066666722297665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,768,0.005964800218741099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,1024,0.006343466540177663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,512,0.005560533205668131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,256,0.005117866893609365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,128,0.004935466746489207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,16384,0.03035306731859843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,12288,0.023730132977167764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,10240,0.020651733875274657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,65536,0.11643093427022297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,8192,0.0173632005850474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,6144,0.013668266932169595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,7168,0.015761066476504007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,5120,0.012010666728019714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,4096,0.01043839951356252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,3584,0.01034986674785614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,2048,0.007778133451938629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,3072,0.009358933568000794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,2560,0.008966400225957235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,1536,0.007249066730340321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,1024,0.0062496001521746315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,768,0.005790933469931285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,16384,0.02969706654548645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,512,0.005452799797058106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,12288,0.023821866512298583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,128,0.004819199939568837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,256,0.005046399931112925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,10240,0.020217599471410115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,8192,0.016643200318018594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,65536,0.10638079643249512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,7168,0.015477333466211954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,3584,0.010256000359853109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,3072,0.009194667140642803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,5120,0.011985066533088683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,6144,0.013381333152453104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,4096,0.010301867127418518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,2560,0.008880000313123066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,2048,0.007692799965540569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,1536,0.007111466427644093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,1024,0.006205866734186808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,768,0.005714133381843567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,512,0.005363200108210246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,256,0.004958933095137278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,16384,0.029074132442474365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,10240,0.019471999009450278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,128,0.004814933240413666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,12288,0.022576000293095908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,7168,0.01530880033969879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,4096,0.010282666484514872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,8192,0.016396799683570863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,6144,0.013315199812253316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,65536,0.1059381326039632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,3584,0.010128000378608703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,5120,0.011974400281906128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,2560,0.008760533730189006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,3072,0.009157333771387737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,2048,0.007630933324495952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,1536,0.0069919998447100324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,1024,0.006076799829800924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,768,0.005684266487757364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,512,0.005420800050099691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,256,0.004902400076389313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,128,0.004727466901143392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,16384,0.028717867533365887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,12288,0.02253440022468567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,8192,0.016369066635767617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,10240,0.019476266702016194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,65536,0.10544533729553222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,7168,0.015310933192571005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,4096,0.010240000486373902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,6144,0.013303466637929282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,3072,0.009105066458384197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,2560,0.008603733777999879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,5120,0.011963733037312825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,2048,0.007650133470694225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,1024,0.0060917332768440245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,3584,0.010024533669153849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,1536,0.007053866485754649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,768,0.005598933498064677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,128,0.004673066735267639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,256,0.004897066454092661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,512,0.005198933184146881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,5120,0.22575786908467613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,7168,0.3201610565185547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,6144,0.27081066767374673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,8192,0.358841609954834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,10240,0.4312138557434082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,12288,0.5204309463500977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,3584,0.15624213218688965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,2048,0.09036586284637452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,3072,0.136246395111084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,4096,0.17906239827473958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,2560,0.1138037363688151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,1536,0.06923627058664958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,1024,0.04832106828689575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,256,0.02987839976946513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,512,0.03511893351872762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,768,0.045365333557128906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,128,0.02756800055503845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,16384,0.6859445571899414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,7168,0.08495360215504964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,6144,0.07409066359202067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,8192,0.09623146851857503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,10240,0.1138965368270874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,12288,0.13576639493306478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,16384,0.1821824073791504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,3584,0.044512001673380534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,4096,0.05118079980214437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,5120,0.06099199851353963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,3072,0.039613866806030275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,2560,0.03362880150477092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,768,0.015110400319099427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,1536,0.02224853237469991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,256,0.01070186694463094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,2048,0.026579199234644572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,128,0.00976746678352356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,1024,0.016899200280507405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,512,0.012834133704503379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,10240,0.1005738655726115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,12288,0.11916266282399494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,7168,0.07517120043436686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,8192,0.08443093299865723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,16384,0.15540800094604493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,6144,0.06474239826202392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,5120,0.05521173477172851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,4096,0.04375146627426148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,3584,0.04027200142542521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,2560,0.029705599943796797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,3072,0.03446186780929565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,2048,0.024357332785924276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,65536,0.6904415766398112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,512,0.010451199611028035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,1024,0.015012266238530478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,1536,0.019826134045918785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,768,0.012981333335240684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,128,0.0077567999561627705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,256,0.008721066514650981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,10240,0.09360960324605307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,16384,0.14461760520935057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,7168,0.07107733090718588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,65536,0.5850517272949218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,12288,0.10992746353149414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,8192,0.08129599889119467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,6144,0.06050453186035156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,3072,0.03292266726493835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,2560,0.028178133567174274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,5120,0.05165760119756063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,4096,0.04244373242060344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,3584,0.036738133430480956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,1536,0.018441599607467652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,2048,0.023001599311828613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,1024,0.01400106648604075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,512,0.009252267082532246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,256,0.007766399780909221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,768,0.012100266416867574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,128,0.007022933165232341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,8192,0.05102399984995524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,10240,0.06101653178532919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,12288,0.0724842627843221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,65536,0.5478687922159831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,16384,0.09507946968078614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,7168,0.044760533173878986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,5120,0.03353600104649861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,3072,0.022882133722305298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,6144,0.040088534355163574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,2560,0.019742933909098308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,4096,0.028150399525960285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,3584,0.025627734263737996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,2048,0.017027199268341064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,256,0.0074986666440963745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,1536,0.014379733800888061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,128,0.0070592001080513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,1024,0.011601066589355469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,768,0.01015786627928416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,512,0.008501332998275758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,7168,0.04198293288548787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,12288,0.06789120038350424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,65536,0.3571701367696126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,10240,0.059203199545542394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,8192,0.0482037345568339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,16384,0.08949759801228842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,3584,0.02425280014673869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,5120,0.03174826701482137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,2560,0.018003199497858682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,4096,0.026357332865397137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,6144,0.036190934975941974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,1024,0.010359467069307963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,3072,0.02095573345820109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,1536,0.01279253363609314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,2048,0.01588053305943807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,768,0.008725333213806152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,256,0.00647680014371872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,512,0.007560533285140991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,128,0.005971199770768484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,8192,0.045569066206614176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,65536,0.3291221300760905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,16384,0.08345920244852702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,7168,0.03961600065231323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,12288,0.06333119869232177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,10240,0.05390079816182455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,6144,0.03430826663970947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,2560,0.01725333333015442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,2048,0.015156267086664834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,5120,0.029710932572682695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,3584,0.022780799865722658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,4096,0.025099732478459674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,3072,0.019718400637308755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,1536,0.012626133362452188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,256,0.00660159985224406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,1024,0.010079999764760334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,768,0.008431999882062276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,512,0.007398400207360585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,128,0.005975466469923655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,65536,0.3003573417663574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,12288,0.06001706520716349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,10240,0.05130773385365804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,16384,0.0793941338857015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,6144,0.03243520061175029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,8192,0.04225706656773885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,7168,0.037611734867095944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,5120,0.027812266349792482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,4096,0.02331519921620687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,3072,0.018947199980417887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,3584,0.02160960038503011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,2560,0.0164490669965744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,2048,0.013990400234858194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,1024,0.009273599584897358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,128,0.005510400235652924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,65536,0.28625494639078775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,1536,0.011622400085131327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,768,0.007957333326339721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,256,0.006126933296521505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,512,0.006955733398596446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,16384,0.06947733561197916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,7168,0.03258453408877055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,12288,0.05442026853561401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,6144,0.028987733523050944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,10240,0.04491413434346517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,8192,0.03668266534805298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,4096,0.021092265844345093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,5120,0.024614399671554564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,3584,0.020032000541687012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,2560,0.015225600202878317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,2048,0.01306880017121633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,3072,0.01790720025698344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,1536,0.011394133170445759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,512,0.007180800040562947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,65536,0.26154026985168455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,1024,0.008726400136947633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,768,0.00788266658782959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,256,0.00639466643333435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,128,0.0058773333827654515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,6144,0.030139732360839843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,12288,0.05546666781107584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,16384,0.07300373713175455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,7168,0.034068266550699874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,8192,0.0383296012878418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,10240,0.0455135981241862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,5120,0.025891200701395674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,3072,0.017383466164271034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,4096,0.02165973385175069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,3584,0.019182932376861573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,1024,0.008273066580295562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,2560,0.015229866902033488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,2048,0.01267093320687612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,1536,0.010398933291435241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,65536,0.24961066246032715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,128,0.005373866856098175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,768,0.0074432000517845156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,512,0.0065290664633115125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,256,0.0059008002281188965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,12288,0.05275733470916748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,6144,0.02883946696917216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,10240,0.04460053443908692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,16384,0.06857066949208578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,8192,0.036744534969329834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,7168,0.033036800225575765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,4096,0.020933334032694498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,5120,0.024142932891845704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,3584,0.019181867440541588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,3072,0.01722666621208191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,2560,0.01453013320763906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,2048,0.012232533097267151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,65536,0.24199040730794272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,1536,0.009830400347709656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,1024,0.00806826651096344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,768,0.007190399865309398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,512,0.006321066617965698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,256,0.005688533186912537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,128,0.005208533505598704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,12288,0.05124693314234415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,16384,0.06602559884389242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,10240,0.043340798219045004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,7168,0.03207466602325439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,8192,0.03524479866027832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,6144,0.02802026669184367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,65536,0.23130879402160645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,5120,0.023539199431737264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,3072,0.016108799974123636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,3584,0.01857066750526428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,4096,0.020426666736602782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,1536,0.009607467055320739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,768,0.007038933535416921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,2560,0.014174933234850565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,2048,0.011527466773986816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,1024,0.007996800045172375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,512,0.0062282666563987735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,256,0.005662933488686879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,128,0.005220266679922739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,12288,0.045413335164388016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,16384,0.05987093448638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,10240,0.0394378662109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,7168,0.02911253372828166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,8192,0.0321834663550059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,6144,0.025489066044489545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,4096,0.018894932667414346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,5120,0.02127573291460673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,65536,0.2154698689778646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,3584,0.016915200153986613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,2560,0.012939733266830445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,3072,0.014881066481272378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,2048,0.01071679989496867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,1536,0.009261866410573322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,128,0.005197866757710775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,1024,0.007660800218582153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,768,0.006969599922498067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,512,0.006163200239340464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,256,0.0056533331672350565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,12288,0.04676906665166219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,16384,0.061190398534139004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,10240,0.03983253240585327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,8192,0.032740267117818196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,7168,0.03012479941050212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,6144,0.026514132817586262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,65536,0.2103754679361979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,5120,0.02185493310292562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,3584,0.016952532529830932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,4096,0.018522665898005167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,3072,0.014493866761525472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,2560,0.012744533022244773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,2048,0.010645332932472228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,1536,0.00915839970111847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,1024,0.0075914666056633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,768,0.006676266590754191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,512,0.006062933305899302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,256,0.005435733497142792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,128,0.005011199911435445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,16384,0.053971199194590244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,12288,0.04243413209915161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,10240,0.03576533397038777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,7168,0.026683733860651655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,6144,0.023524266481399537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,8192,0.03015039960543315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,65536,0.18800959587097169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,2048,0.01011306643486023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,5120,0.019805866479873657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,4096,0.016528000434239708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,3584,0.015121066570281982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,3072,0.01328213314215342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,2560,0.011866666873296102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,1536,0.008858666817347208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,1024,0.007334400216738384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,768,0.006587733328342438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,512,0.0059690664211908976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,128,0.004947199920813243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,256,0.005350400010744731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,16384,0.044719998041788736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,12288,0.035341866811116535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,10240,0.02951253255208333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,65536,0.16321172714233398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,8192,0.024903466304143272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,7168,0.022080000241597494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,6144,0.01957226594289144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,5120,0.01646719972292582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,4096,0.014470400412877402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,3584,0.013247999548912048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,3072,0.011617066462834676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,2048,0.008931199709574383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,2560,0.010338133573532105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,1536,0.008075733482837678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,1024,0.006628266473611195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,768,0.006186666587988535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,16384,0.03293333252271016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,256,0.005193600058555603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,512,0.00565119981765747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,128,0.004857600231965383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,12288,0.026738133033116656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,5120,0.012829867005348206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,4096,0.011452800035476685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,65536,0.12415253321329753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,10240,0.02194026708602905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,8192,0.018004266421000163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,7168,0.016315733393033348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,6144,0.014620799819628397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,2560,0.008764800429344178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,768,0.005846400062243144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,3584,0.01088746686776479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,3072,0.009476266304651896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,128,0.00480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,1536,0.007170133292675018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,1024,0.006306133170922597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,2048,0.007717333237330119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,512,0.0054666668176651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,12288,0.021331199010213218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,256,0.005011199911435445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,16384,0.02721280058224996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,10240,0.018539732694625853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,8192,0.015494400262832641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,65536,0.0990453322728475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,7168,0.014316800236701965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,5120,0.011307733257611592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,3584,0.010070400436719258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,3072,0.008798933029174805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,6144,0.012771200140317282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,4096,0.010206933816274006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,512,0.005323733389377594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,768,0.005625600119431814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,2560,0.008261333405971526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,2048,0.0073066666722297665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,256,0.004922666649023692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,1536,0.006931200126806895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,1024,0.006006399790445963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,8192,0.01544533371925354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,10240,0.018497065703074137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,12288,0.021322667598724365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,128,0.0047189335028330484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,65536,0.09749653339385986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,16384,0.026844799518585205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,4096,0.010173867146174114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,7168,0.014294399817784628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,6144,0.012739200393358865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,2048,0.007142400244871776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,5120,0.011358933647473653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,1536,0.007002666592597961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,3584,0.009852799773216247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,1024,0.0060362666845321655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,3072,0.008649599552154542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,2560,0.008133333424727123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,256,0.00489386667807897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,768,0.00553599993387858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,512,0.005203199883302053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,128,0.004659200211366018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,6144,0.3057344118754069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,5120,0.2545909404754639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,7168,0.35514561335245765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,8192,0.4027509371439616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,3584,0.1776565392812093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,10240,0.47333653767903644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,3072,0.14809813499450683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,4096,0.19765973091125488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,2560,0.12564906279246013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,12288,0.5421791712443034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,512,0.03462186654408773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,1536,0.07772800127665201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,128,0.026948267221450807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,1024,0.05389120181401571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,256,0.029745066165924074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,768,0.0451477328936259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,2048,0.10238826274871826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,16384,0.7010527928670247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,7168,0.09299946626027425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,8192,0.10442773501078288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,10240,0.12815786997477213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,12288,0.14518507321675617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,6144,0.0817845344543457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,16384,0.1895253340403239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,5120,0.06837226549784342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,4096,0.055078399181365964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,1536,0.02331519921620687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,3584,0.05003413359324137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,2560,0.0357749342918396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,3072,0.04210026661554973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,512,0.013366400202115377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,256,0.010811733206113179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,2048,0.030114134152730305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,128,0.009777067104975383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,1024,0.017516799767812095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,768,0.015269333124160766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,8192,0.09225066502888998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,7168,0.0818890651067098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,12288,0.12608959674835205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,10240,0.11172693570454914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,16384,0.16657172838846843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,5120,0.06090773344039917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,4096,0.048341333866119385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,3584,0.043450665473937986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,6144,0.06968959967295328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,3072,0.03733546733856201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,768,0.013602133591969809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,65536,0.70348695119222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,2560,0.03299946586290996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,1536,0.02073813279469808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,2048,0.02686506708463033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,1024,0.015687466661135355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,128,0.007772799829641979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,512,0.010666666428248088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,256,0.008796800176302593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,12288,0.1203925371170044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,10240,0.10597973664601643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,16384,0.1549173355102539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,7168,0.07779946327209472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,8192,0.08905173142751058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,65536,0.6210720062255859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,3072,0.03544853528340657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,6144,0.06569386720657348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,4096,0.04553066492080689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,3584,0.04223786592483521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,2048,0.025814400116602583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,5120,0.05804053147633871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,2560,0.031142399708429976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,1536,0.019228800137837728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,1024,0.01487573285897573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,768,0.012745599945386252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,512,0.010090667009353637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,256,0.007749333480993907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,128,0.00703359991312027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,16384,0.10009386539459228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,12288,0.0791040023167928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,7168,0.04813333352406819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,10240,0.06777919928232828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,8192,0.05463253259658814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,65536,0.5750570933024088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,6144,0.04243733485539754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,3584,0.027115732431411743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,3072,0.023964800437291465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,5120,0.036108799775441486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,4096,0.030635732412338256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,1024,0.012001066406567892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,2560,0.021016534169514975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,2048,0.01773759921391805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,256,0.007559466858704884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,128,0.006960000097751618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,512,0.008691199620564779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,1536,0.014787200093269347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,768,0.01004266639550527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,8192,0.05094826618830363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,10240,0.0631381352742513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,65536,0.3629173278808594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,12288,0.0735647996266683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,7168,0.04533333381017049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,4096,0.027931733926137285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,16384,0.09726826349894205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,3584,0.024932267268498738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,1536,0.013291733463605246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,6144,0.03883626858393351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,5120,0.03384106556574504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,3072,0.022272000710169472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,2560,0.019409066438674925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,768,0.008866133292516072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,2048,0.016622933745384216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,512,0.007747200131416321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,256,0.00655680000782013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,1024,0.010636799534161885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,128,0.0059338668982187905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,10240,0.06059733231862387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,65536,0.3415210723876953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,8192,0.04774080117543538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,12288,0.06965013345082602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,7168,0.04123946825663249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,5120,0.03128319978713989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,16384,0.09212160110473633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,3584,0.02441706657409668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,2048,0.01585706671079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,3072,0.02071146567662557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,6144,0.03687680164972941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,2560,0.018619734048843383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,4096,0.026361600557963057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,1024,0.01050879955291748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,1536,0.013054933150609335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,768,0.008820266524950663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,512,0.007547733187675476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,256,0.006640000144640605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,128,0.005946666498978933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,65536,0.3187146822611491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,16384,0.08320426940917969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,12288,0.06578773260116577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,8192,0.04487359921137492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,10240,0.05698026816050211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,4096,0.02518613338470459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,6144,0.03483733336130778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,7168,0.0391701340675354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,5120,0.029998934268951415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,2048,0.014882133404413859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,3584,0.022711465756098427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,2560,0.0175872008005778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,65536,0.30671253204345705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,3072,0.019874133666356406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,1536,0.012134400010108948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,1024,0.00941439966360728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,128,0.005544533332188925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,768,0.008151466647783916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,512,0.007028266787528992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,256,0.006182399888833364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,7168,0.034993068377176924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,10240,0.04938666820526123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,12288,0.059077334403991696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,8192,0.03995413382848104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,16384,0.07269866466522217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,6144,0.030701865752538044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,5120,0.02621866663297017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,4096,0.022490666309992472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,2560,0.016155733664830526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,3584,0.02143146594365438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,1536,0.01151146690050761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,3072,0.017797333002090455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,65536,0.2802997271219889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,2048,0.01356160044670105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,1024,0.009116799632708231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,768,0.008054399987061818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,512,0.007294933497905731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,256,0.0064650664726893115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,128,0.0060138667623202005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,8192,0.041620266437530515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,16384,0.07582506338755289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,12288,0.060045866171518955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,10240,0.05192853212356567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,7168,0.03591786623001099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,6144,0.03264426589012146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,4096,0.02330346703529358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,5120,0.026935466130574542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,3584,0.021257599194844566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,3072,0.018119466304779053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,2048,0.013664000233014426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,1536,0.010786133011182149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,2560,0.01601599951585134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,65536,0.271395206451416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,768,0.00761599987745285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,1024,0.008775466680526733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,256,0.005892266829808554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,512,0.006708266834417979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,128,0.005376000205675761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,10240,0.05151679913202921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,8192,0.04011413256327311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,16384,0.07464000384012857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,7168,0.034932267665863034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,12288,0.05822933514912924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,6144,0.03095146616299947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,3584,0.020933334032694498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,4096,0.022827732563018798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,5120,0.02611093322436015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,2048,0.012878933548927307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,3072,0.017464532454808553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,2560,0.016006400187810264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,65536,0.25994879404703775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,1536,0.010642133156458537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,128,0.005179733534653981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,1024,0.008444799979527792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,768,0.00726506660381953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,512,0.006495999793211619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,256,0.005736533304055532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,7168,0.03464853366216024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,6144,0.03049280047416687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,8192,0.03883093198140462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,12288,0.057708799839019775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,16384,0.07292906443277994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,10240,0.04935466845830282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,4096,0.021897600094477335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,5120,0.025634133815765382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,3584,0.020345600446065266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,65536,0.24907414118448892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,3072,0.017347200711568197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,2560,0.015128533045450846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,2048,0.012742400169372559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,1536,0.010125866532325745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,1024,0.008249600231647492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,512,0.006439466774463653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,768,0.007211733361085255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,256,0.005694933235645294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,128,0.005269333223501841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,10240,0.043492265542348224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,7168,0.031566933790842695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,16384,0.06403626600901285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,12288,0.0504800001780192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,8192,0.03510826826095581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,6144,0.028257066011428834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,5120,0.022702932357788086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,65536,0.2364415963490804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,4096,0.01996906598409017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,3584,0.01801919937133789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,3072,0.014894933501879374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,1024,0.007941333452860515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,1536,0.009890133142471313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,2560,0.013740799824396768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,2048,0.011896533767382304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,768,0.00720000018676122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,16384,0.06773013273874919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,512,0.006390400230884552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,256,0.005696000158786773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,128,0.005217066903909048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,10240,0.045109331607818604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,12288,0.05282346804936727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,8192,0.03745599985122681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,6144,0.028692267338434857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,7168,0.03222613334655762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,4096,0.021316266059875487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,5120,0.02546986738840739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,3584,0.018667733669281004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,65536,0.23044800758361816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,2048,0.011553066968917846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,3072,0.01618559956550598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,2560,0.014150399963061014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,1024,0.007925333579381307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,1536,0.00972160001595815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,768,0.006938666601975759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,512,0.006171733140945435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,256,0.005499733487764994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,16384,0.06163946787516276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,128,0.005015466610590617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,10240,0.04119360049565633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,12288,0.04754879872004191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,8192,0.031845333178838094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,65536,0.2006293296813965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,7168,0.02808213432629903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,5120,0.021197867393493653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,3584,0.017620267470677693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,6144,0.026476800441741943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,4096,0.01885546644528707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,768,0.006899199883143107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,2560,0.012694399555524191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,3072,0.013955199718475341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,2048,0.010662399729092916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,256,0.0053951998551686605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,1536,0.009290666381518046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,128,0.004897066454092661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,1024,0.007652266820271809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,512,0.006026666859785716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,16384,0.04964906771977742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,65536,0.18127573331197103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,8192,0.027616000175476073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,12288,0.039628799756368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,10240,0.03376533190409343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,7168,0.02477653423945109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,6144,0.021575466791788737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,4096,0.01572160025437673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,5120,0.01821546753247579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,3072,0.012242133418718975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,3584,0.014383999506632486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,2560,0.011315199732780456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,2048,0.009611733754475911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,1024,0.00697920024394989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,1536,0.008365866541862488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,512,0.005764266848564148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,10240,0.02726080020268758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,768,0.006402133405208588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,12288,0.03123626708984375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,256,0.005257600049177805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,128,0.0049333333969116214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,65536,0.14353386561075848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,8192,0.022334933280944824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,5120,0.01474453310171763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,16384,0.04085439840952555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,7168,0.01915093262990316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,4096,0.012682666381200155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,3072,0.010259200135866802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,2048,0.008205866813659668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,6144,0.017670400937398276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,768,0.005861333509286245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,1024,0.0062837332487106325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,3584,0.011760000387827556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,2560,0.009062400460243225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,1536,0.007523199915885926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,512,0.005389866729577383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,256,0.005096533397833506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,128,0.004884266853332519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,65536,0.10197866757710775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,16384,0.026115200916926068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,12288,0.02122559944788615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,10240,0.018052266041437785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,5120,0.011319466431935628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,4096,0.0100682665904363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,7168,0.0137855996688207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,8192,0.015238400300343832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,6144,0.012391466895739238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,2560,0.008226133386294047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,3072,0.008501332998275758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,2048,0.0071733335653940845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,3584,0.009815466403961182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,1024,0.00603413333495458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,128,0.004729599754015604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,768,0.0056415999929110205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,12288,0.020615466435750327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,1536,0.006886399785677592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,16384,0.025967999299367266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,512,0.0052928000688552855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,256,0.004969599843025208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,10240,0.017819732427597046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,65536,0.09483199914296468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,6144,0.012417067090670269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,3584,0.009594666957855224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,7168,0.013716266552607218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,5120,0.01134399970372518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,4096,0.009975467125574749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,3072,0.008318933347860973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,8192,0.015118933717409768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,2560,0.008157866696516674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,1024,0.006011733412742614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,768,0.00557226687669754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,512,0.005288533369700114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,256,0.004856533308823904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,1536,0.00692799985408783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,128,0.004648533463478088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,2048,0.007088000078996022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,5120,0.12646186351776123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,6144,0.14830826123555502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,8192,0.1914463996887207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,7168,0.1696074644724528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,4096,0.10417706966400146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,10240,0.23440000216166176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,3072,0.08247786362965902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,2560,0.07080960273742676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,3584,0.0936074654261271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,12288,0.27874666849772134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,1536,0.05101973215738932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,768,0.031540266672770184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,2048,0.05906240145365397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,512,0.024563199281692503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,1024,0.03587520122528076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,256,0.018959999084472656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,128,0.016804265975952148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,16384,0.36368319193522136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,10240,0.06626773277918498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,8192,0.05510079860687256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,6144,0.049165864785512284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,7168,0.04956266482671102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,12288,0.07837440172831217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,16384,0.10072746276855468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,2048,0.019986132780710854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,4096,0.03249066670735677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,3072,0.02629759907722473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,2560,0.023373866081237794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,3584,0.029628799359003706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,5120,0.03825386762619019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,1024,0.012997333208719888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,768,0.011667199929555257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,512,0.010071466366449993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,256,0.007609599828720092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,1536,0.01632960041364034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,128,0.007143466671307881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,12288,0.061393066247304284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,7168,0.039100801944732665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,16384,0.0790005366007487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,8192,0.0437120000521342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,10240,0.052198398113250735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,5120,0.03025173346201579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,4096,0.025573333104451496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,2560,0.018158932526906334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,6144,0.03475519816080729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,3584,0.023434666792551677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,1024,0.0107722669839859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,3072,0.02064853310585022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,768,0.009668266773223877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,2048,0.015332266688346863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,1536,0.01301653285821279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,512,0.007221333185831706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,65536,0.3718613306681315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,256,0.006478933493296306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,128,0.0061471998691558834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,12288,0.05273919900258382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,8192,0.037752532958984376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,10240,0.05082773367563883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,7168,0.033744001388549806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,16384,0.06789120038350424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,65536,0.28821118672688806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,5120,0.026050132513046265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,3072,0.017781333128611247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,6144,0.02998720010121663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,2560,0.017273600896199545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,3584,0.020372267564137778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,4096,0.022218666474024453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,1024,0.009783466657002766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,512,0.00659093310435613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,2048,0.013608533143997192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,128,0.005707733333110809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,1536,0.011634133259455363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,768,0.008451199531555176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,256,0.006044800082842508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,65536,0.24517439206441244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,10240,0.03828373352686564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,8192,0.03181013266245524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,7168,0.028523733218510944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,16384,0.0565280000368754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,12288,0.04475520054499309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,6144,0.028402133782704668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,3072,0.015129599968592325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,5120,0.022394667069117226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,3584,0.01763946612675985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,4096,0.019030400117238364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,2560,0.014081066846847535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,2048,0.012377599875132244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,65536,0.2051626682281494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,1024,0.009065600236256917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,768,0.00763626645008723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,512,0.006603733201821645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,256,0.006175999840100607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,128,0.005864533285299936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,1536,0.010919466614723206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,6144,0.023375999927520753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,12288,0.040788265069325765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,7168,0.026283733050028485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,8192,0.029158399502436323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,10240,0.03497920036315918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,16384,0.05194559892018637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,4096,0.017422932386398315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,3072,0.013948800166447959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,3584,0.015564800302187601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,2560,0.012862933675448099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,5120,0.020371200640996297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,2048,0.011272533734639486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,65536,0.1890944004058838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,1536,0.009841066598892213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,768,0.006691200037797292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,512,0.00625493327776591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,128,0.005525333185990652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,256,0.005832533538341522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,1024,0.00768746683994929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,6144,0.02107200026512146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,7168,0.02355519930521647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,8192,0.026530132691065474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,12288,0.036673065026601157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,10240,0.031666133801142374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,16384,0.04731626510620117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,3072,0.012769066294034324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,4096,0.015396266182263692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,5120,0.01856213410695394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,3584,0.01423466702302297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,2560,0.011801600456237793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,2048,0.010450133681297302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,65536,0.1715221405029297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,1536,0.009091200431187947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,256,0.005444266895453135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,128,0.005312000215053558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,512,0.005829333265622457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,768,0.006243200103441874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,1024,0.0070720002055168155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,6144,0.019362133741378785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,10240,0.029054933786392213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,12288,0.03348586559295654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,7168,0.021819732586542764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,8192,0.024158932765324912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,16384,0.04292906522750854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,5120,0.017004799842834473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,65536,0.15580159823099773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,3072,0.011942399541536967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,4096,0.014430933197339377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,2560,0.011100799838701884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,3584,0.013265066345532737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,2048,0.009976533055305482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,1536,0.008381866415341695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,1024,0.006965333223342895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,128,0.005285333096981049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,512,0.005904000004132589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,768,0.006420266628265381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,256,0.0054848000407218935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,6144,0.016556800405184428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,7168,0.01835839947064718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,8192,0.020618667205174766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,16384,0.036030932267506914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,12288,0.02823893427848816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,10240,0.024174932638804117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,65536,0.1349621295928955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,5120,0.014636799693107605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,4096,0.01279039978981018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,3072,0.010726400216420491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,2048,0.00885973374048869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,3584,0.011893332997957865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,2560,0.010263466835021972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,1536,0.00782400021950404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,1024,0.0064629331231117245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,256,0.005364266534646353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,512,0.005826133489608765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,768,0.006072533130645752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,128,0.005169066786766052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,16384,0.033998934427897136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,6144,0.015592533349990844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,8192,0.019051732619603474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,12288,0.026653865973154705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,7168,0.01753066579500834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,10240,0.02299626668294271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,65536,0.12480533123016357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,5120,0.013988266388575235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,4096,0.012235732873280843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,3584,0.011621333161989848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,3072,0.010396800438563029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,2048,0.008515200018882752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,2560,0.009846400221188862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,1536,0.007619200150171916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,1024,0.00621013343334198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,768,0.006018133461475372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,128,0.005129600067933401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,512,0.005731200178464254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,256,0.005317333340644837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,16384,0.03550933202107747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,12288,0.02709439992904663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,6144,0.016075733304023742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,10240,0.023483733336130776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,7168,0.01759786605834961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,8192,0.01945706605911255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,65536,0.1292522668838501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,5120,0.014379733800888061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,4096,0.01258133351802826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,3072,0.010419199864069622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,3584,0.011617066462834676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,2560,0.009357866644859315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,2048,0.0083146666487058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,1536,0.007125333448251088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,128,0.005102933446566264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,768,0.005987200140953064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,512,0.005702400207519531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,1024,0.006580266853173573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,256,0.005306666592756907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,16384,0.03402986526489258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,12288,0.02592960000038147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,6144,0.015582933028539022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,10240,0.022528000672658286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,8192,0.01881386637687683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,7168,0.017132800817489625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,65536,0.12568746407826742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,5120,0.014141866564750671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,4096,0.012261333068211873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,3072,0.010062932968139648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,2560,0.009211732943852743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,1536,0.007085866729418437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,3584,0.011238400141398113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,2048,0.008041599889596303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,1024,0.006459733347098033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,256,0.005177600185076395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,512,0.005751466751098633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,768,0.005929600199063619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,128,0.005066666503747304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,16384,0.028579199314117433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,12288,0.022088533639907836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,8192,0.01599253316720327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,10240,0.019205333789189656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,6144,0.013758933544158936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,7168,0.014883200327555338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,65536,0.10436480045318604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,5120,0.012482133507728577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,2048,0.007322666545708974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,3072,0.008901333808898926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,4096,0.010963199536005656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,3584,0.009993599851926167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,1536,0.007036800185839335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,2560,0.008368000388145447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,1024,0.006239999830722809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,512,0.005479466418425242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,128,0.004947199920813243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,768,0.005791999896367391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,256,0.005153066913286845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,16384,0.027101866404215497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,12288,0.02090239922205607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,65536,0.09896426995595296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,8192,0.015598932902018229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,7168,0.014541866381963095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,10240,0.018626133600870766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,6144,0.013405866424242654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,5120,0.012102400263150532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,4096,0.010322133700052898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,2048,0.007207466661930085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,3072,0.008384000261624653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,3584,0.009638399879137675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,2560,0.008235733211040496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,1024,0.0060479998588562015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,1536,0.00697920024394989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,768,0.005676800012588501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,10240,0.018198400735855103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,16384,0.026306132475535076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,12288,0.02076266606648763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,128,0.004878933231035868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,512,0.005387733379999796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,256,0.004990933338801066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,7168,0.01434346636136373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,6144,0.012759466965993246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,4096,0.009867733716964722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,8192,0.015213867028554281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,2560,0.008145066599051159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,65536,0.10121920108795165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,5120,0.011101866761843365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,3584,0.009436800082524618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,3072,0.008337066570917765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,512,0.005201066533724466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,2048,0.007084799806276958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,1536,0.006929066777229309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,768,0.005577600002288819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,1024,0.005946666498978933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,256,0.004930133124192556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,128,0.0047872001926104225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,12288,0.02064746618270874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,16384,0.025869866212209065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,10240,0.01814613342285156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,7168,0.013686399658521017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,8192,0.01516800026098887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,65536,0.09063999652862549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,6144,0.012625066439310708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,5120,0.011145599683125814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,4096,0.009789866209030152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,3584,0.009434666236241658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,2048,0.007065600156784058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,3072,0.008285866677761078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,2560,0.008057599763075511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,128,0.004693333307902018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,1536,0.0068234667181968685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,1024,0.006001066664854685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,512,0.005378133555253347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,768,0.005592533449331919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,256,0.0048981333772341405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,12288,0.020456532637278237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,10240,0.017509333292643228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,7168,0.013429333766301474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,6144,0.012382933497428894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,16384,0.025806933641433716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,8192,0.01460693379243215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,2560,0.008053333560625712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,65536,0.09074880282084147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,2048,0.007086933155854543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,5120,0.01118293305238088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,4096,0.009754666686058044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,3584,0.009444266557693481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,1024,0.005914666752020518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,3072,0.0082997332016627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,256,0.004863999783992767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,512,0.0052149335543314615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,1536,0.006826666494210561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,12288,0.019703465700149535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,768,0.005588266750176748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,16384,0.024977066119511924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,10240,0.01725546717643738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,128,0.004724266628424326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,8192,0.014569600423177084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,7168,0.013411200046539307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,4096,0.009834667046864826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,2560,0.008002133170763651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,65536,0.08999359607696533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,6144,0.01229866643746694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,2048,0.007080533107121785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,5120,0.01108906666437785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,1536,0.00673280010620753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,3584,0.009327999750773112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,3072,0.008286933104197185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,1024,0.005864533285299936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,512,0.005196799834569296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,768,0.0054613331953684485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,128,0.004725333551565806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,256,0.004817066589991251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,16384,0.024969599644343057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,65536,0.08942399819691976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,6144,0.012353066603342693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,5120,0.011117866635322571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,12288,0.01975359916687012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,8192,0.014568533500035605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,10240,0.017299199104309083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,3584,0.00923520028591156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,4096,0.009741866588592529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,7168,0.013432533542315165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,3072,0.00827946662902832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,2560,0.007900799810886382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,768,0.005467733244101206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,512,0.0051136001944541935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,1536,0.006819200019041698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,256,0.00488319993019104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,2048,0.007029333213965099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,128,0.004691199958324432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,1024,0.005925333499908448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,5120,0.12423573335011799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,6144,0.14559787114461262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,7168,0.16678080558776856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,8192,0.18795413970947267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,10240,0.23312640190124512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,4096,0.10213332970937092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,3584,0.0917685349782308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,3072,0.08066133658091226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,2048,0.06066346565882365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,12288,0.27727041244506834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,1024,0.03465706507364909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,2560,0.06970132986704508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,512,0.023468800385793052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,768,0.03091199994087219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,128,0.01604159971078237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,1536,0.04607679843902588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,256,0.018119466304779053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,16384,0.3606815973917643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,8192,0.05445866584777832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,7168,0.04851839939753215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,10240,0.06530239979426065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,12288,0.07693119843800864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,16384,0.09839359919230142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,4096,0.03173653284708659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,6144,0.048186667760213214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,5120,0.03718506495157878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,1024,0.012955733140309653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,3584,0.028885332743326823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,3072,0.025733333826065064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,2048,0.019513599077860513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,768,0.01148906648159027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,256,0.007570133109887441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,1536,0.015847466389338174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,512,0.009781333804130554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,2560,0.022858667373657226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,128,0.006874666611353557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,7168,0.03842666546503703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,10240,0.052102398872375486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,8192,0.043211734294891356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,16384,0.07825067043304443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,12288,0.061281065146128334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,6144,0.03402986526489258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,2560,0.017844265699386595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,3584,0.02288960019747416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,5120,0.02955946723620097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,4096,0.02536533276240031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,3072,0.020307199160257975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,1536,0.012804266810417176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,1024,0.010633599758148194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,512,0.007506133119265239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,256,0.006491733094056447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,2048,0.015424000223477683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,128,0.005994666616121928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,768,0.00946666697661082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,65536,0.3657514572143555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,7168,0.03681173324584961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,16384,0.07043733596801757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,10240,0.04659413496653239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,12288,0.05441279808680216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,8192,0.0380789319674174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,65536,0.29951146443684895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,6144,0.029620265960693358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,4096,0.021966934204101562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,5120,0.026131200790405273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,3584,0.02008426586786906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,1536,0.01149120032787323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,3072,0.017900800704956053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,2560,0.01694719990094503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,2048,0.013636266191800436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,256,0.006029866635799408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,1024,0.009709866841634114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,768,0.007915733257929485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,128,0.00561706672112147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,512,0.006997333467006683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,65536,0.2607658704121908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,10240,0.040727468331654866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,8192,0.03292693297068278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,12288,0.04696213404337565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,7168,0.029335467020670573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,16384,0.06072426637013754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,3072,0.016023466984430947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,3584,0.018017067511876424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,4096,0.01952426632245382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,6144,0.027881600459416706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,5120,0.023104000091552734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,2560,0.01430293321609497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,1536,0.010855467120806376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,768,0.007904000083605449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,1024,0.008892800410588582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,2048,0.012390399972597759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,512,0.007062399884064992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,256,0.006346666812896728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,128,0.005875200033187866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,16384,0.0626261313756307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,65536,0.22613439559936524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,8192,0.03321066697438558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,10240,0.04057919979095459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,12288,0.047006932894388835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,7168,0.02916266719500224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,3072,0.015610667069753012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,5120,0.02315733234087626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,6144,0.02583786646525065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,3584,0.017100799083709716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,4096,0.018693333864212035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,2560,0.013875200351079305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,512,0.0065749332308769224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,2048,0.01190826694170634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,1024,0.008296533425649007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,256,0.00582826683918635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,1536,0.010290132959683736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,768,0.0071168000499407455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,65536,0.22572372754414877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,128,0.0053375999132792154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,10240,0.03824426730473836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,12288,0.04408426682154338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,8192,0.031568000713984176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,16384,0.05990826686223348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,7168,0.02839786609013875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,4096,0.01755839983622233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,6144,0.024655999739964803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,5120,0.021130667130152384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,3584,0.016204800208409628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,3072,0.014747732877731323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,2560,0.013056000073750814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,2048,0.011245866616566975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,1536,0.00985599954922994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,512,0.006248533229033152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,1024,0.00763626645008723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,768,0.0068800002336502075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,65536,0.2143808046976725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,256,0.005627733469009399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,128,0.005148800214131674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,6144,0.023549866676330567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,16384,0.056695465246836343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,12288,0.04240213235219319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,5120,0.020463999112447104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,7168,0.026608000199000042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,8192,0.030506666501363116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,10240,0.03667093515396118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,3584,0.015537066260973611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,4096,0.017422932386398315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,2560,0.012574932972590127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,2048,0.010986666878064473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,3072,0.014371200402577718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,1536,0.009516800443331402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,512,0.006196266909440359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,65536,0.20433707237243653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,1024,0.007522133489449819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,768,0.006963199873765309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,256,0.0055978665749231975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,128,0.00526506652434667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,7168,0.024027733008066814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,16384,0.046728531519571945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,8192,0.02717120051383972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,10240,0.0318069338798523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,12288,0.03744959831237793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,6144,0.020782933632532755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,5120,0.01817173361778259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,4096,0.015614933768908181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,3072,0.01346986691157023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,2560,0.011826133728027344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,3584,0.01471466620763143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,2048,0.010408533612887065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,65536,0.1903978665669759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,1536,0.008602666854858398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,1024,0.007180800040562947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,768,0.006712533533573151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,256,0.00556160012880961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,128,0.005117866893609365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,512,0.006179200112819671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,16384,0.045954132080078126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,12288,0.03563946485519409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,7168,0.023203200101852416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,8192,0.02589226762453715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,6144,0.020615466435750327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,10240,0.030954666932423908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,4096,0.015074132879575094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,65536,0.1692512035369873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,3584,0.014176000157992044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,3072,0.012869333227475485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,5120,0.018292266130447387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,2560,0.011766399939854939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,2048,0.009800533453623455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,1536,0.008278400202592214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,1024,0.006986666719118755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,512,0.006001066664854685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,768,0.006537599861621857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,128,0.005198933184146881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,256,0.0055402666330337524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,10240,0.03370026747385661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,8192,0.027165865898132323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,7168,0.02413866718610128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,6144,0.021090133984883627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,16384,0.048087465763092044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,12288,0.03774720033009847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,4096,0.015520000457763672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,5120,0.018614399433135986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,3072,0.013205333550771078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,3584,0.014318933089574179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,2560,0.011769599715868632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,2048,0.009991466999053955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,65536,0.18827625910441081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,1536,0.008391466736793519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,768,0.0066101332505544026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,1024,0.007255466779073079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,256,0.005352533360322317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,512,0.0060586666067441305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,128,0.004965333143870035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,8192,0.025500800212224322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,12288,0.03512320121129354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,10240,0.031380265951156616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,6144,0.020141865809758505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,7168,0.022870399554570518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,16384,0.04448000192642212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,65536,0.16612159411112468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,5120,0.01814080079396566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,3584,0.013497599959373474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,4096,0.014503467082977294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,3072,0.012036266922950744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,2560,0.01062506635983785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,2048,0.009249066313107807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,1536,0.007843199868996937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,1024,0.006896000107129414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,768,0.006275199850400289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,128,0.004999466737111409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,512,0.0058005332946777345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,256,0.005252266426881155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,12288,0.033012266953786215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,10240,0.02929919958114624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,16384,0.04149440129597982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,8192,0.024562132358551026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,7168,0.02190933426221212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,6144,0.01967786749204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,3584,0.013315199812253316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,65536,0.16659199396769206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,5120,0.016773333152135216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,3072,0.011770666639010111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,4096,0.014533332983652749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,2560,0.010514133175214132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,2048,0.009133866429328919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,1536,0.008083199958006541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,1024,0.006926933427651723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,768,0.006307200094064077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,256,0.005217066903909048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,512,0.0058005332946777345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,128,0.00491839994986852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,10240,0.026074665784835815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,16384,0.03789866765340169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,6144,0.017357865969340004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,12288,0.02947733402252197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,7168,0.019683200120925903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,8192,0.02174826661745707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,5120,0.01535040040810903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,65536,0.14315199851989746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,4096,0.012777599692344665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,3584,0.011774933338165284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,3072,0.010483200351397198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,2560,0.009692800045013428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,2048,0.008471467097600301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,1536,0.007417599856853485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,1024,0.0064629331231117245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,768,0.0059445331494013464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,512,0.005550933380921682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,256,0.005128533144791921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,128,0.00490880012512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,16384,0.03164586623509725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,8192,0.01813439925511678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,7168,0.016272000471750894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,12288,0.023931733767191567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,10240,0.02230400045712789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,4096,0.010182399551073711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,65536,0.1092842658360799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,2560,0.00842026670773824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,6144,0.014086400469144186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,5120,0.012311466534932454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,3584,0.009731200337409974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,3072,0.00881599982579549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,512,0.0052928000688552855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,2048,0.007469866673151653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,1536,0.006910933554172516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,1024,0.005961599946022034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,768,0.005716266731421152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,256,0.00494400014479955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,128,0.004830933113892873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,8192,0.016370133558909098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,65536,0.10715093612670898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,16384,0.02871573368708293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,12288,0.02049493392308553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,10240,0.0197653333346049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,5120,0.010796800255775452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,4096,0.00942186713218689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,7168,0.014122666915257773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,6144,0.01204800009727478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,1536,0.006795733173688252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,3584,0.009341866771380106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,3072,0.008237866560618083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,2560,0.008022400240103405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,2048,0.007075199981530507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,256,0.004880000154177348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,1024,0.005960533519585928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,768,0.0054666668176651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,512,0.005236266553401947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,10240,0.017030400037765504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,8192,0.01453013320763906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,128,0.004732800026734671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,7168,0.01325440009435018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,12288,0.019875200589497884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,16384,0.02528746724128723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,3584,0.009245866537094116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,65536,0.09365866978963217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,6144,0.011932800213495891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,2560,0.007922133306662242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,5120,0.010542933146158855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,4096,0.009316266576449076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,1536,0.006726400057474773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,1024,0.005919999877611796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,3072,0.008177066842714947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,2048,0.006994133194287618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,128,0.0047541335225105286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,16384,0.02407360076904297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,768,0.005543466905752818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,512,0.005133866767088572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,256,0.004860800007979075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,65536,0.08630826473236083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,8192,0.014404267072677612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,6144,0.011932800213495891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,7168,0.013273599743843078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,12288,0.01920213301976522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,5120,0.010583466291427613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,10240,0.016621866822242738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,3584,0.009231999516487122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,2048,0.0070154666900634766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,4096,0.009307733178138733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,512,0.00518506666024526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,3072,0.008213333288828532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,2560,0.007916800181070964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,1024,0.0058890665570894875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,1536,0.00664106657107671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,768,0.005381333331267038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,10240,0.016638933618863424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,16384,0.024091732501983643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,256,0.00487253318230311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,128,0.004640000065167745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,7168,0.013267200191815695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,5120,0.010564266641934713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,12288,0.019211733341217042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,4096,0.009296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,8192,0.014441600441932679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,65536,0.08526079654693604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,3584,0.009046399593353271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,3072,0.00810346653064092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,2560,0.0078015998005867004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,6144,0.011907200018564861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,1536,0.006865066786607106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,1024,0.005830400188763936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,256,0.004794666667779287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,512,0.005072000126043955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,2048,0.006902400155862172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,128,0.004643199841181437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,768,0.005363200108210246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,5120,0.12823039690653484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,6144,0.1477952003479004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,7168,0.16822506586710612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,8192,0.19271039962768555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,10240,0.24142826398213707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,4096,0.10146986643473307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,3584,0.09063786665598551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,12288,0.282258129119873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,3072,0.07992640336354574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,1536,0.04567573467890422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,1024,0.03399146795272827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,256,0.017465599377950034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,2560,0.06885440349578857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,768,0.03199359973271688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,2048,0.06006933450698852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,512,0.02294506629308065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,128,0.01573973298072815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,16384,0.36734078725179037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,6144,0.04790079991022746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,8192,0.05408213138580322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,16384,0.09849387009938558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,7168,0.04822826782862345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,10240,0.06602666775385538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,12288,0.07780799865722657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,5120,0.03682560125986735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,2048,0.019232000907262167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,2560,0.022588799397150673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,3072,0.025520000855127973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,3584,0.028549333413441975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,1536,0.01585493286450704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,4096,0.031524266799290976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,768,0.011384532848993937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,256,0.007531733314196269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,128,0.006775466601053874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,1024,0.012748799721399941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,512,0.009697066744168599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,8192,0.044392534097035724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,12288,0.06397653420766194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,7168,0.039077333609263104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,10240,0.05463146766026815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,16384,0.08072853088378906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,6144,0.03459733327229818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,3584,0.023209599653879802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,4096,0.025032534201939897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,5120,0.03017279903093974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,3072,0.020566399892171225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,2560,0.01812053322792053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,2048,0.015459199746449789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,1536,0.012682666381200155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,1024,0.010620799660682679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,512,0.007548800110816956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,768,0.009377066294352214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,256,0.006562133133411407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,128,0.005846400062243144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,65536,0.37994667689005535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,12288,0.05892906586329142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,8192,0.040498133500417074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,10240,0.05039360125859579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,16384,0.07630933125813802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,7168,0.038344534238179524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,3072,0.018736000855763754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,5120,0.02767893274625142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,4096,0.023057067394256593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,6144,0.031040000915527343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,3584,0.02122559944788615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,2560,0.016338133811950685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,1536,0.011541333794593812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,65536,0.2986805280049642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,768,0.00856213370958964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,512,0.007043200234572093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,256,0.006236800054709116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,2048,0.014055466651916504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,1024,0.009641599655151368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,128,0.005504000186920166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,12288,0.050977067152659095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,7168,0.030103466908137005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,8192,0.03327573339144389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,10240,0.04309226671854655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,16384,0.06482239961624145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,65536,0.27210667928059895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,6144,0.026708267132441205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,5120,0.023125332593917847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,3584,0.019063466787338258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,4096,0.020100265741348267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,3072,0.016389333208402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,2048,0.013031466801961263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,2560,0.015497600038846334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,256,0.006438399851322174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,128,0.00584853341182073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,1536,0.01102293332417806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,1024,0.009134933352470398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,768,0.008377599716186523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,512,0.00714026689529419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,65536,0.24608960151672363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,10240,0.044939732551574706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,12288,0.052001067002614344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,8192,0.03566933472951253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,7168,0.03324906627337138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,16384,0.06469120184580485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,4096,0.020516266425450645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,6144,0.028987733523050944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,3584,0.018736000855763754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,2560,0.015169066190719605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,5120,0.02437439958254496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,3072,0.016914133230845133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,2048,0.01283093293507894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,512,0.0066549330949783325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,1536,0.010968533158302308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,1024,0.008558932940165203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,768,0.00767573316891988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,128,0.005310933291912079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,256,0.0059349333246548975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,12288,0.04944426616032918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,65536,0.24597652753194174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,16384,0.06546773513158163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,5120,0.023560533920923867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,10240,0.04323199987411499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,8192,0.033913600444793704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,7168,0.0320032000541687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,2560,0.01444586714108785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,6144,0.026229333877563477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,4096,0.01982933282852173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,3584,0.018090667327245076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,3072,0.016296533743540446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,1024,0.008024533092975617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,768,0.0074314668774604796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,2048,0.012379733721415202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,1536,0.010175999999046326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,65536,0.2384864012400309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,512,0.006424533327420552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,128,0.00514026681582133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,256,0.005672533313433329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,16384,0.06383253335952759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,6144,0.025290666023890178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,8192,0.03281173308690389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,12288,0.04704533418019612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,10240,0.0413045326868693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,7168,0.02910826603571574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,5120,0.022369066874186196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,3584,0.017560533682505288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,2048,0.011961600184440613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,4096,0.01930026610692342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,3072,0.015795200069745382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,2560,0.013982933759689332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,65536,0.2348682721455892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,1536,0.00995733340581258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,128,0.00525546669960022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,1024,0.007886933286984761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,768,0.0073066666722297665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,256,0.005723733206590017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,512,0.006312533219655354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,6144,0.021691733598709108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,12288,0.04022186597188314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,7168,0.0248799999554952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,16384,0.053453866640726724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,8192,0.027550933758417766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,10240,0.035382401943206784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,3584,0.015425067146619162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,5120,0.018614399433135986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,4096,0.016754132509231568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,65536,0.21362880071004234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,3072,0.013946666320164999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,2560,0.012219732999801636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,2048,0.011006933450698853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,1536,0.008945066730181377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,256,0.005650133391221364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,1024,0.007451733450094859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,768,0.007222400108973186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,512,0.006312533219655354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,128,0.005134933193524678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,12288,0.0395082672437032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,16384,0.04938559929529826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,8192,0.027556266387303668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,10240,0.035189334551493326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,7168,0.025883734226226807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,6144,0.02148373325665792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,4096,0.016759467124938966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,5120,0.019529600938161217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,2048,0.010660266876220703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,1536,0.00853546659151713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,3584,0.015466666221618653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,65536,0.18484053611755372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,2560,0.01234773298104604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,3072,0.013957333564758301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,768,0.00705813318490982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,1024,0.007299200197060903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,512,0.006179200112819671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,256,0.005602133274078369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,10240,0.038840532302856445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,128,0.005151999990145365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,12288,0.043622398376464845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,16384,0.05466986497243246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,6144,0.02469653288523356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,7168,0.02723520000775655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,8192,0.030401066939036055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,4096,0.017989333470662436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,5120,0.02016213337580363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,3584,0.01636799971262614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,3072,0.01471466620763143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,2048,0.010955733060836793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,2560,0.013133866588274637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,1536,0.008736000458399455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,65536,0.21340266863505045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,1024,0.007457066575686138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,256,0.005547733108202616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,12288,0.04099839925765991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,512,0.006181333462397257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,768,0.007020799815654755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,128,0.005039999882380167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,10240,0.0360213319460551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,16384,0.05345173279444376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,7168,0.02648213307062785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,8192,0.02902719974517822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,65536,0.18542720476786295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,6144,0.0225055992603302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,5120,0.019778132438659668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,4096,0.016525866587956746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,3584,0.01560640037059784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,3072,0.014098133643468222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,2048,0.010044800241788228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,2560,0.011609599987665812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,1536,0.008413867155710856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,1024,0.0072736000021298725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,768,0.006679466863473256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,512,0.0060149331887563075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,256,0.005323733389377594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,10240,0.03255893389383952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,128,0.004934399823347728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,12288,0.03678400119145711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,16384,0.04683200120925903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,8192,0.027827199300130206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,6144,0.020165334145228066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,5120,0.017372800906499227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,7168,0.024513065814971924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,65536,0.1805749257405599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,4096,0.015667200088500977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,1536,0.008332799871762593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,3584,0.01385599970817566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,3072,0.012344533205032348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,2560,0.011002666751543681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,2048,0.009685333569844563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,1024,0.007165866593519847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,768,0.006906666855017345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,512,0.005987200140953064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,16384,0.04365866581598918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,256,0.005306666592756907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,128,0.00489386667807897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,8192,0.02539520064989726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,12288,0.03465493520100911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,10240,0.030015999078750612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,7168,0.0219050665696462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,6144,0.01934079925219218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,5120,0.017262933651606242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,65536,0.1563360055287679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,3584,0.013079466422398886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,4096,0.014126933614412942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,2560,0.010407466689745586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,1536,0.007968000074227651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,3072,0.0116074671347936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,2048,0.008969600001970928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,1024,0.006728533407052357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,768,0.006353066861629486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,512,0.005715199808279673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,256,0.005203199883302053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,8192,0.02065599958101908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,7168,0.01858560045560201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,128,0.0048426667849222815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,16384,0.037247999509175615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,12288,0.02826026678085327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,10240,0.025778132677078246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,3584,0.010815999905268351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,6144,0.0156768004099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,65536,0.12495253086090088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,5120,0.01372053325176239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,2560,0.009227733810742695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,4096,0.011403733491897583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,2048,0.008003200093905132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,1536,0.00701333334048589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,3072,0.009820800026257832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,1024,0.006233599781990051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,256,0.005008000135421753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,128,0.004816000163555145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,768,0.005831466615200042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,16384,0.03118399977684021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,10240,0.020547199249267577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,512,0.005420800050099691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,7168,0.014813866217931113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,65536,0.10445226828257244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,6144,0.01317759950955709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,5120,0.011667199929555257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,12288,0.02411093314488729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,4096,0.009781333804130554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,8192,0.017336533466974894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,3072,0.008356266220410665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,3584,0.009301333626111349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,1536,0.006702933212121327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,768,0.0056618665655454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,1024,0.005922133227189382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,256,0.0049216002225875854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,2560,0.008202666540940602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,2048,0.007156266768773396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,12288,0.020253866910934448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,512,0.00522986650466919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,16384,0.02595840096473694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,8192,0.014339199662208557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,128,0.004727466901143392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,65536,0.09505706628163656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,6144,0.011615999539693196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,4096,0.009250133236249288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,10240,0.01686186591784159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,2560,0.007922133306662242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,7168,0.01304213305314382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,5120,0.010549333691596986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,1024,0.005814399818579356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,3584,0.009172266721725464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,3072,0.008146133522192638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,256,0.0048650667071342465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,512,0.0052159999807675685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,2048,0.006974933544794719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,1536,0.006660266717274983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,768,0.005454933146635691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,16384,0.02378773291905721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,12288,0.018773333231608073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,65536,0.08415040175120035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,6144,0.01157973309357961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,128,0.004769066472848257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,4096,0.009249066313107807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,3072,0.008108800152937572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,8192,0.014264532923698425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,7168,0.013018666704495748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,10240,0.016581333676973977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,5120,0.01050986647605896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,2560,0.007834666470686594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,1536,0.00655680000782013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,3584,0.00909866690635681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,2048,0.006974933544794719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,1024,0.0058037335673968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,768,0.005345066885153452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,128,0.004633600016434988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,256,0.004858666658401489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,12288,0.018810667594273887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,10240,0.016569599509239197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,16384,0.02372373342514038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,8192,0.014252799749374389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,512,0.00518506666024526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,5120,0.01048959990342458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,4096,0.009174399574597676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,65536,0.0834933360417684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,3584,0.009048533439636231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,7168,0.013037866353988648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,2048,0.006898133456707001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,6144,0.011566932996114094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,1536,0.006708266834417979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,256,0.004773333172003428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,3072,0.008070399860541026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,128,0.004628266890843709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,2560,0.007719466586907704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,1024,0.005782400071620941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,512,0.0051136001944541935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,768,0.005338666836420695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,5120,0.12061866919199626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,6144,0.1424938678741455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,7168,0.16340160369873047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,8192,0.1849237283070882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,10240,0.2317514737447103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,12288,0.2719711939493815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,4096,0.09926826953887939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,3072,0.07674559752146402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,3584,0.08786026636759439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,2048,0.0581279993057251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,2560,0.0656063993771871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,16384,0.35663038889567056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,128,0.01242453356583913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,256,0.01437226633230845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,512,0.02081706722577413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,768,0.02616853316624959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,1024,0.032232532898585006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,1536,0.0477514664332072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,8192,0.0521717349688212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,7168,0.046480000019073486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,12288,0.07479146321614584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,10240,0.06310720046361287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,6144,0.041048534711201984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,16384,0.09577066898345947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,5120,0.0356383999188741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,4096,0.02993706663449605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,3072,0.02423786719640096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,2048,0.01802240014076233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,3584,0.02718186577161153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,2560,0.021393066644668578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,128,0.005929600199063619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,256,0.006239999830722809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,512,0.008692266543706258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,768,0.010282666484514872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,1024,0.011593600114186604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,65536,0.357585080464681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,1536,0.014874666929244995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,12288,0.05802559852600098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,8192,0.04111253420511882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,10240,0.04922346671422322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,16384,0.07430613040924072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,6144,0.032278400659561154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,7168,0.041178667545318605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,3072,0.019181867440541588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,2560,0.0168938676516215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,5120,0.02807146708170573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,4096,0.023800534009933472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,3584,0.0215829332669576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,2048,0.014268799622853597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,128,0.005167999863624572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,1536,0.011841066678365071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,512,0.0061365331212679545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,256,0.005568000177542368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,768,0.008653866251309712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,1024,0.009757866462071735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,65536,0.27383359273274743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,7168,0.03164373238881429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,12288,0.049702401955922446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,8192,0.0353877345720927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,10240,0.042430933316548666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,6144,0.03127253254254659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,16384,0.06341439882914225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,3072,0.016552533706029257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,2560,0.014492799838383993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,3584,0.018588799238204955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,4096,0.020805333058039346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,2048,0.012638933459917703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,5120,0.024449066321055094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,1536,0.010775466759999592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,256,0.005224533379077911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,128,0.005039999882380167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,512,0.005776000022888183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,768,0.00745066652695338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,1024,0.009037866195042928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,65536,0.23195947011311852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,8192,0.029729066292444865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,12288,0.04171839952468872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,10240,0.03548053503036499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,6144,0.026375466585159303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,7168,0.026700800657272337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,16384,0.052332798639933266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,3072,0.01411946713924408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,4096,0.01748159925142924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,2048,0.011374933520952861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,5120,0.020623999834060668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,3584,0.016182399789492288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,2560,0.013009066383043924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,65536,0.1890944004058838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,512,0.005690666536490122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,256,0.005358933409055074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,128,0.005133866767088572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,768,0.006442666550477346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,1024,0.008091733356316884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,1536,0.009980799754460652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,6144,0.02165973385175069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,7168,0.02444266676902771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,8192,0.0272053341070811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,10240,0.032535467545191446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,12288,0.03788160085678101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,16384,0.04803733428319295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,3072,0.012935466567675271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,4096,0.01623040040334066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,3584,0.014491732915242514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,2560,0.011881599823633831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,5120,0.018838399648666383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,2048,0.010533333818117777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,65536,0.170853328704834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,128,0.004866133133570353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,256,0.005075199902057648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,768,0.006119466821352641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,1536,0.009143466750780743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,512,0.0054730668663978575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,1024,0.00703359991312027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,6144,0.01957546671231588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,12288,0.033855998516082765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,10240,0.02885546684265137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,7168,0.02191466689109802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,8192,0.024493867158889772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,16384,0.04257386525472005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,3072,0.011920000116030376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,5120,0.017053866386413576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,4096,0.01437333325544993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,3584,0.013219199577967324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,2560,0.0109333336353302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,2048,0.009729066491127014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,65536,0.15113174120585124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,512,0.005222400029500326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,768,0.00553706685702006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,128,0.0047882666190465295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,256,0.004966400067011515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,1536,0.008594133456548055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,1024,0.006221866607666016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,6144,0.017516799767812095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,12288,0.030371199051539104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,7168,0.01976319948832194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,10240,0.02601066629091899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,8192,0.02195733388264974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,16384,0.03810773293177287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,65536,0.133733336130778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,5120,0.015230933825174967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,4096,0.013138133287429809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,3072,0.011065600315729777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,2048,0.009186133742332458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,2560,0.01028053363164266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,3584,0.012156800429026286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,1536,0.008231466511885326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,768,0.005493333439032236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,512,0.005177600185076395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,256,0.004935466746489207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,1024,0.005875200033187866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,128,0.004775466521581014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,16384,0.03145066698392232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,7168,0.016375466187795003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,6144,0.014615466197331747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,8192,0.018270933628082277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,10240,0.02179946700731913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,12288,0.025332266092300416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,65536,0.10884586970011394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,5120,0.013057066996892294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,4096,0.011570133765538533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,2048,0.008155733346939087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,3072,0.010075733065605164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,2560,0.009533866246541341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,3584,0.011012267072995503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,1536,0.00673280010620753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,256,0.004907733201980591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,512,0.0052597333987553915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,128,0.004752000172932943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,1024,0.005730133255322774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,768,0.005515733361244201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,16384,0.0291978657245636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,6144,0.013827199737230936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,8192,0.01704533298810323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,7168,0.015305599570274353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,10240,0.020308266083399452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,12288,0.023596799373626708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,65536,0.10274133682250977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,5120,0.012389333049456278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,2048,0.007382399837176005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,4096,0.01095146636168162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,3072,0.009614933530489604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,3584,0.010589866836865743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,2560,0.009071999788284301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,1536,0.006682666639486949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,1024,0.005690666536490122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,768,0.005433600147565206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,128,0.004705066482226053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,256,0.004907733201980591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,512,0.005208533505598704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,16384,0.029629866282145183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,6144,0.013772799571355184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,10240,0.020454400777816774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,12288,0.023618133862813313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,7168,0.015369600057601929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,8192,0.016926934321721397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,65536,0.1045535961786906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,5120,0.012521599729855856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,4096,0.010948266585667927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,3072,0.009320533275604248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,3584,0.010296533505121868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,2048,0.007190399865309398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,2560,0.008295466502507527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,1536,0.00624533345301946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,1024,0.005705599983533224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,128,0.0046858668327331545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,256,0.004857600231965383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,768,0.005276800195376078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,512,0.0050570666790008545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,16384,0.02789013385772705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,12288,0.022336000204086305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,10240,0.019191465775171914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,8192,0.01623253325621287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,6144,0.013264000415802002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,7168,0.014707199732462563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,65536,0.09944852987925211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,5120,0.01204159955183665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,3072,0.008802133798599242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,4096,0.010665599505106609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,1536,0.0061482667922973635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,2560,0.007916800181070964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,2048,0.007129600147406261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,3584,0.01007466713587443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,1024,0.005739733576774597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,768,0.005297066768010458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,256,0.004829866687456766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,128,0.004626133541266123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,512,0.005046399931112925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,16384,0.02331519921620687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,12288,0.018496000766754152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,65536,0.08205653031667073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,10240,0.016396799683570863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,8192,0.014293332894643148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,7168,0.012845866878827415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,6144,0.011742933591206869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,5120,0.010647466778755188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,4096,0.009525332848230999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,3072,0.008302933474381765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,3584,0.008939733107884724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,2048,0.006499200065930684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,2560,0.007249066730340321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,1536,0.006200533111890157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,1024,0.00543146679798762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,768,0.005196799834569296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,512,0.004993066688378652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,256,0.00481386681397756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,128,0.004636799792448679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,12288,0.01809920072555542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,16384,0.022272000710169472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,10240,0.015980799992879234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,65536,0.07833706537882487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,8192,0.013928533593813578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,7168,0.012596266468365988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,6144,0.011504000425338745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,5120,0.010334933797518413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,3584,0.008346666892369587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,4096,0.008830933769543966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,3072,0.007551999886830647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,2048,0.006439466774463653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,2560,0.007242666681607564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,1536,0.006101333101590474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,1024,0.005380266904830932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,768,0.005160533388455709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,512,0.004957866668701172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,256,0.004763733347256978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,128,0.004572799801826477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,16384,0.02185493310292562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,12288,0.01797440052032471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,65536,0.07624213695526123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,10240,0.015850667158762613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,8192,0.013773866494496665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,7168,0.011987200379371643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,6144,0.011250133315722149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,5120,0.009573333462079366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,4096,0.00851200024286906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,3072,0.007454933226108551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,3584,0.008301866551240284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,2560,0.007195733487606049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,2048,0.006389333307743073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,1536,0.006095999975999197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,1024,0.00537066658337911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,768,0.005099733173847198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,512,0.00490880012512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,256,0.004611200094223023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,128,0.004496000210444133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,16384,0.021911466121673585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,12288,0.01788053313891093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,65536,0.07570239702860514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,10240,0.015800533692042033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,8192,0.013199999928474426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,7168,0.011870933572451274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,6144,0.010631466905275982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,5120,0.0095551997423172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,4096,0.008482133348782856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,3072,0.007484800120194752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,2048,0.006362666686375936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,2560,0.007134933272997539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,3584,0.008252800007661184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,1536,0.0060703997810681665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,1024,0.0053151999910672505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,256,0.004628266890843709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,768,0.005121066669623057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,128,0.004474666714668274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,512,0.004870399832725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,16384,0.021798400084177653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,65536,0.07507839997609457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,12288,0.0173632005850474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,10240,0.014872533082962037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,7168,0.011553066968917846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,6144,0.010474666953086853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,8192,0.01281599998474121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,5120,0.009553066889444987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,4096,0.008492799599965413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,3584,0.008280533552169799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,3072,0.007418666779994964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,2048,0.006431999802589417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,2560,0.007125333448251088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,1536,0.0060928001999855045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,1024,0.005312000215053558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,768,0.005095466474692027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,512,0.0047882666190465295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,256,0.004620799918969473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,128,0.00448639988899231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,65536,0.07410453160603842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,16384,0.020898133516311646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,12288,0.01694399913152059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,10240,0.014906666676203408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,8192,0.0127893328666687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,7168,0.011539199948310852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,6144,0.010451199611028035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,5120,0.009477333227793375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,4096,0.008474666873613994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,3584,0.00817920019229253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,3072,0.007464533547560374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,2560,0.00710399995247523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,2048,0.006366933385531108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,1536,0.006087466577688853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,1024,0.005272533496220907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,768,0.005032533407211303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,256,0.004613333443800608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,128,0.004499199986457825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,512,0.004856533308823904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,16384,0.020881066719690956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,65536,0.07446613311767578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,12288,0.01689493258794149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,10240,0.01483626663684845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,8192,0.012880000472068786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,7168,0.011479467153549194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,6144,0.010472533106803895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,5120,0.009474133451779682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,4096,0.008437333504358928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,3584,0.00823359986146291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,3072,0.0074080000321070354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,2048,0.00644053320089976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,2560,0.007136000196139018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,1536,0.006110933423042297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,1024,0.005217066903909048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,768,0.005051733553409576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,512,0.004795733094215393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,256,0.004610133171081543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,128,0.004453333218892416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,5120,0.11935573418935139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,6144,0.1409066677093506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,7168,0.1618677298227946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,8192,0.18512852986653644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,10240,0.22732267379760743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,12288,0.27031574249267576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,4096,0.09806506633758545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,2560,0.06471253236134847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,3584,0.08689173062642416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,3072,0.076036270459493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,1536,0.04687466621398926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,2048,0.05726293325424194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,1024,0.031593600908915206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,16384,0.355077330271403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,768,0.0256117324034373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,512,0.020132267475128175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,256,0.014176000157992044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,128,0.012133333086967468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,6144,0.04068586826324463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,7168,0.046062934398651126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,8192,0.051363201936086025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,10240,0.06294933160146078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,12288,0.08478720188140869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,16384,0.09493546485900879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,5120,0.035095465183258054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,3584,0.02687679926554362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,4096,0.029625600576400755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,3072,0.02381333311398824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,2560,0.021203200022379555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,1024,0.011566932996114094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,2048,0.017678932348887125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,1536,0.01476479967435201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,512,0.00865280032157898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,768,0.010198400417963664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,256,0.006382933259010315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,128,0.005849599838256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,10240,0.048503466447194415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,12288,0.05705920060475668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,8192,0.0404853343963623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,7168,0.03605120182037354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,16384,0.07350293000539145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,5120,0.027848533789316815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,6144,0.03202773332595825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,4096,0.023548799753189086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,3584,0.02129813234011332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,3072,0.019040000438690186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,65536,0.35455894470214844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,2048,0.01428053379058838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,2560,0.016666666666666666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,1536,0.011678933103879293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,1024,0.009751466910044353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,768,0.008697600166002909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,512,0.006640000144640605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,256,0.0056096002459526065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,128,0.005284266670544943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,65536,0.2707658767700195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,8192,0.03504213492075602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,10240,0.04160319964090983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,12288,0.04900799989700318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,7168,0.031243733565012616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,16384,0.06263039906819662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,6144,0.03096533417701721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,3072,0.016314666469891867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,5120,0.024072533845901488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,4096,0.02040106654167175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,2560,0.014514133334159851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,3584,0.0184608002503713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,768,0.00745066652695338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,512,0.005975466469923655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,2048,0.012537599603335062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,1536,0.010644267002741497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,1024,0.008946133653322856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,256,0.005341866612434387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,128,0.005018666883309682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,65536,0.22962346076965331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,10240,0.034959999720255534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,12288,0.04092053174972534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,8192,0.029371732473373414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,16384,0.0517525315284729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,7168,0.026195200284322102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,3584,0.016057599584261575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,4096,0.017382399241129557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,2560,0.012922666470209756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,3072,0.014086400469144186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,5120,0.020261333386103312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,6144,0.026152533292770386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,2048,0.0113045334815979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,1536,0.009928533434867859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,1024,0.007973333199818928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,512,0.005955199897289276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,768,0.006682666639486949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,256,0.005530666808287303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,128,0.005220266679922739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,65536,0.18633599281311036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,16384,0.04772053162256877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,10240,0.03211413423220317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,12288,0.03707093397776286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,8192,0.026825600862503053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,6144,0.023945599794387817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,7168,0.02399466633796692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,4096,0.015943466623624166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,2560,0.011865599950154623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,5120,0.018632533152898152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,3584,0.014338133732477823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,3072,0.012785067160924276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,2048,0.01045973300933838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,1536,0.008900266885757447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,1024,0.006771199901898702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,768,0.0061951999862988796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,512,0.005605333546797434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,256,0.0051807999610900875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,128,0.004911999901135763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,65536,0.16804693539937338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,16384,0.04195520083109538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,12288,0.03344746828079224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,10240,0.0286901334921519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,8192,0.02400319973627726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,7168,0.02164586583773295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,6144,0.02153279980023702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,2560,0.010914132992426554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,5120,0.016746666034062704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,4096,0.014272000392278036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,3584,0.013054933150609335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,3072,0.011918933192888895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,2048,0.009629866480827332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,65536,0.14870400428771974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,1536,0.008428800106048583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,1024,0.006466133395830791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,768,0.006083199878533682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,512,0.005496533215045929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,256,0.005130666494369507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,128,0.004811733464399974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,10240,0.02563626567522685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,6144,0.017451733350753784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,8192,0.021673599878946938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,12288,0.029687466224034627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,7168,0.019637332359949747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,16384,0.03776426712671916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,5120,0.01493013302485148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,4096,0.012964266538619994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,3072,0.011040000120798747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,3584,0.012077866991360982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,2560,0.01027733286221822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,2048,0.009084799885749817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,1536,0.007683200140794118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,128,0.004829866687456766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,1024,0.006668800115585327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,65536,0.13284053007761637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,768,0.006002133091290792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,512,0.005389866729577383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,256,0.005015466610590617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,7168,0.01644266645113627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,16384,0.03187519907951355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,10240,0.02376746733983358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,8192,0.018054399887720743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,12288,0.025118933121363325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,6144,0.015032533804575601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,5120,0.013198933005332947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,3584,0.011058132847150166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,3072,0.010084266463915508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,4096,0.011642666657765706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,2560,0.009506133198738099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,2048,0.008191999793052674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,65536,0.1128480037053426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,256,0.005026133358478546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,1536,0.006938666601975759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,1024,0.006049066781997681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,768,0.00602346658706665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,512,0.005416533350944519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,128,0.004861866434415182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,16384,0.029674667119979858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,8192,0.016753067572911583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,10240,0.020194133122762047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,12288,0.023187200228373207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,7168,0.0151936004559199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,6144,0.01397333343823751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,65536,0.10457386970520019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,5120,0.012389333049456278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,4096,0.011016533772150675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,3584,0.010505599776903789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,3072,0.009614933530489604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,2560,0.009065600236256917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,2048,0.007776000102361043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,512,0.005319466690222422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,1536,0.006714666883150737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,1024,0.005868799984455109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,768,0.005858133236567179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,256,0.004935466746489207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,128,0.004782933493455251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,10240,0.020297600825627645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,16384,0.030102399984995525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,8192,0.017030400037765504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,7168,0.015452800194422403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,12288,0.02355413238207499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,6144,0.014132266243298849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,4096,0.0110944002866745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,5120,0.012549333771069846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,65536,0.10833066304524738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,3584,0.010316800077756245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,3072,0.009523199995358785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,2560,0.008642133076985676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,2048,0.007268266876538594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,1536,0.006467199822266896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,512,0.005275733272234599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,1024,0.005994666616121928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,128,0.004728533327579498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,768,0.005787733197212219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,256,0.004969599843025208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,12288,0.021924267212549843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,10240,0.018866133689880372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,8192,0.016184533635775246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,16384,0.02836266756057739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,7168,0.014828800161679586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,3584,0.010030933221181234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,6144,0.013315199812253316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,5120,0.012115200360616047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,4096,0.010617599884668986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,3072,0.008958933750788371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,65536,0.1006229321161906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,2560,0.007705600063006084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,1536,0.006222933530807495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,768,0.005640533566474914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,2048,0.0069365332523981735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,1024,0.005845333139101664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,512,0.005100800096988678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,256,0.004874666531880697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,128,0.004705066482226053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,12288,0.020173867543538414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,16384,0.026237867275873822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,10240,0.01713706652323405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,5120,0.011264000336329143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,8192,0.014447999993960061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,7168,0.01346560021241506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,6144,0.012746666868527731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,65536,0.09134720166524252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,4096,0.009948800007502239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,3584,0.009266133109728496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,3072,0.008281599978605907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,2560,0.00757120003302892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,2048,0.006868266562620799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,1536,0.0063296000162760425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,1024,0.005686399837334951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,768,0.005785599847634633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,512,0.0051125332713127134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,256,0.0048426667849222815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,128,0.004747733473777771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,12288,0.01795413295427958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,16384,0.022476800282796226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,10240,0.015642666816711427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,8192,0.013585066795349121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,65536,0.07952000300089518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,7168,0.012520533800125123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,6144,0.011523200074831645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,5120,0.010353066523869832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,4096,0.00881813367207845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,3072,0.007629866898059845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,3584,0.008376533786455791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,2560,0.007209600011507671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,2048,0.006550399959087372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,1536,0.006083199878533682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,1024,0.005418666700522105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,512,0.0050122668345769245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,768,0.005432533224423727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,256,0.00480320006608963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,128,0.004644266764322917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,16384,0.02169813315073649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,12288,0.017578667402267455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,10240,0.015284267067909241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,7168,0.01195199986298879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,65536,0.07430613040924072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,8192,0.013184000054995218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,6144,0.010769066214561463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,5120,0.009705600142478944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,2048,0.0064181332786877945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,4096,0.008402132987976074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,3584,0.008225066463152568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,2560,0.0071274667978286745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,3072,0.007401599983374278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,512,0.004894933104515076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,1024,0.005345066885153452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,1536,0.005947733422120413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,768,0.005261866748332978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,256,0.004640000065167745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,128,0.004561066627502441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,16384,0.02176106572151184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,12288,0.01753600041071574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,10240,0.015064533551534018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,65536,0.07341759999593099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,8192,0.0129120002190272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,7168,0.01162453293800354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,6144,0.01039466659228007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,5120,0.009633066256841023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,4096,0.008372267087300617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,3584,0.008222933113574981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,3072,0.007406933108965556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,2560,0.0071168000499407455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,2048,0.006361599763234456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,1536,0.005926399926344554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,256,0.0046293333172798155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,1024,0.005288533369700114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,768,0.005054933329423269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,512,0.004890666902065277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,128,0.004568533102671305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,16384,0.02172693411509196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,12288,0.01694186727205912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,65536,0.07305599848429362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,10240,0.01458560029665629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,8192,0.012327466408411663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,7168,0.011424000064531963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,6144,0.01034879982471466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,5120,0.009380267063776652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,3584,0.008191999793052674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,3072,0.007406933108965556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,4096,0.008404266834259034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,2560,0.00699946681658427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,256,0.004586666822433472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,2048,0.0063178668419520065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,1536,0.005922133227189382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,1024,0.005258666475613912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,768,0.005130666494369507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,512,0.004801066716512045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,128,0.0044832001129786175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,16384,0.021100799242655434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,12288,0.016586666305859886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,65536,0.0722261349360148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,10240,0.014281599720319112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,8192,0.012335999806722005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,7168,0.011374933520952861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,6144,0.010358400146166484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,5120,0.009409067034721375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,3584,0.008140799899895985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,4096,0.008427733182907104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,3072,0.007348266740640004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,2560,0.006985599795977275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,2048,0.00631573349237442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,1536,0.0059456000725428265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,1024,0.005187200009822845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,768,0.005011199911435445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,256,0.004586666822433472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,512,0.00486826648314794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,128,0.00447680006424586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,65536,0.0725279966990153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,16384,0.020768000682195028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,10240,0.014275200168291726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,12288,0.016537599762280784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,8192,0.012345600128173827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,7168,0.01135040024916331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,6144,0.010378666718800863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,5120,0.00939626693725586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,4096,0.008294400076071422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,3584,0.008142933249473572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,3072,0.007362133264541626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,2560,0.007045333087444305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,2048,0.006330666442712148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,1536,0.005886933207511902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,1024,0.005224533379077911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,768,0.005016533533732096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,512,0.004769066472848257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,256,0.00461760014295578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,128,0.004401066899299621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,5120,0.11824853420257568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,6144,0.1395232041676839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,7168,0.16120533943176268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,8192,0.18188479741414387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,10240,0.22535360654195152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,12288,0.2691136042277018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,4096,0.09668479760487875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,2560,0.06383359829584757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,3072,0.07514560222625732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,3584,0.08558826446533203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,2048,0.05669973293940226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,768,0.024998400608698526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,16384,0.3528821309407552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,1536,0.04611626863479614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,1024,0.030756266911824544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,512,0.019399466117223103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,256,0.014482133587201438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,128,0.011937066912651062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,7168,0.045507200558980304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,8192,0.05066666603088379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,6144,0.04002773364384969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,10240,0.06210986773173014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,12288,0.07321706612904867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,16384,0.10916799704233807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,5120,0.03465066750844319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,3072,0.023579732577006022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,4096,0.02913813392321269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,3584,0.026610134045283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,1536,0.014501333236694336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,2560,0.021441066265106203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,2048,0.01758079926172892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,128,0.0058335999647776285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,1024,0.011425066987673442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,768,0.010276266932487487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,512,0.008412800232569377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,256,0.006404266754786174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,12288,0.056689067681630456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,8192,0.04006826480229696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,7168,0.03563626607259114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,10240,0.048070398966471355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,16384,0.07245439688364665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,6144,0.0315615991751353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,4096,0.023312000433603923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,5120,0.027471999327341717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,3072,0.018809600671132406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,3584,0.02105600039164225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,65536,0.35172799428304036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,1024,0.009666132926940917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,2560,0.01658560037612915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,2048,0.014196266730626425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,1536,0.01169599990049998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,768,0.00864746669928233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,512,0.006673066814740498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,256,0.0056970665852228795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,128,0.005154133339722951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,65536,0.2678773244222005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,8192,0.034663466612497966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,12288,0.04935146570205688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,10240,0.041405868530273435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,16384,0.062334934870402016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,7168,0.030925865968068438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,6144,0.0306442658106486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,3072,0.01646613379319509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,3584,0.01844480037689209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,5120,0.023731199900309245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,4096,0.02020053267478943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,2560,0.01439573367436727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,2048,0.012444800138473511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,1536,0.01053546667098999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,1024,0.008914132912953694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,768,0.007447466750939687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,512,0.006214400132497152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,256,0.005422933399677277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,128,0.005015466610590617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,8192,0.029734400908152263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,16384,0.05229760011037191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,10240,0.03543039957682292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,65536,0.22996053695678711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,12288,0.041092268625895184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,7168,0.0262773334980011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,3584,0.017708800236384072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,4096,0.017442133029301962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,6144,0.02345386743545532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,5120,0.020426666736602782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,2048,0.01114453375339508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,1536,0.009827199578285217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,3072,0.014146133263905843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,1024,0.007867733140786488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,768,0.006897066533565521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,2560,0.012928000092506409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,512,0.00621973325808843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,256,0.005658666789531708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,128,0.005217066903909048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,8192,0.029577600955963134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,12288,0.03970239957173665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,65536,0.19013439814249675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,16384,0.05115413268407186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,10240,0.03439679940541585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,7168,0.025443200270334882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,3072,0.013578666249910989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,3584,0.015267200271288552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,4096,0.017534933487574258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,2560,0.0123690664768219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,6144,0.02262506683667501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,5120,0.019618133703867592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,2048,0.010705066720644633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,1536,0.009241599837938945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,256,0.005300266544024149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,128,0.004850133260091146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,1024,0.007630933324495952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,768,0.006438399851322174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,512,0.005827199916044871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,65536,0.1856053352355957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,10240,0.032348799705505374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,12288,0.03730346759160359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,16384,0.048266665140787764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,7168,0.024318933486938477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,8192,0.02757333318392436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,6144,0.02200746734937032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,5120,0.01834133267402649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,2048,0.01011946698029836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,4096,0.015602133671442666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,3584,0.014401066303253173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,3072,0.01304533382256826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,2560,0.011832533280054729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,65536,0.17107733090718585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,1536,0.00888320008913676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,1024,0.006985599795977275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,768,0.00626453310251236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,512,0.005787733197212219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,256,0.00521919975678126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,128,0.004837333162625631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,16384,0.045713067054748535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,6144,0.020467199881871543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,7168,0.02308053374290466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,12288,0.035230934619903564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,8192,0.026317866643269856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,10240,0.030896000067392987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,3072,0.012397866447766621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,2560,0.011361066500345867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,2048,0.009835732976595561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,5120,0.017313067118326822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,65536,0.1652458667755127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,1536,0.008241066833337148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,4096,0.014978133638699851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,3584,0.01381119986375173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,512,0.005658666789531708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,1024,0.006856533388296763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,768,0.0060576001803080235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,256,0.0052042668064435325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,128,0.004772266745567322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,12288,0.03208746711413066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,7168,0.020322134097417198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,5120,0.01599679986635844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,16384,0.04280960162480672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,10240,0.02846933404604594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,8192,0.023404800891876222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,4096,0.013621333241462707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,6144,0.01858560045560201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,3072,0.01186240017414093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,65536,0.143668270111084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,3584,0.01276586651802063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,2560,0.010681600371996561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,2048,0.009256533781687419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,1536,0.008087466657161712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,768,0.006089599927266439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,1024,0.006525866687297821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,128,0.004822400212287903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,256,0.005106133222579956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,512,0.005667200187842051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,12288,0.029726932446161907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,5120,0.015331199765205384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,8192,0.022642133633295695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,7168,0.019320533672968546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,16384,0.039043200016021726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,10240,0.026088533798853557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,6144,0.017517866690953572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,3072,0.01120960017045339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,65536,0.13360533714294434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,4096,0.013267200191815695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,2048,0.008338133494059246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,3584,0.012412800391515096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,2560,0.010204799969991048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,1536,0.007250133156776428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,1024,0.00631466656923294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,512,0.00547733356555303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,768,0.005913599828879039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,256,0.005070933202902476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,128,0.004823466638724009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,10240,0.027552000681559247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,16384,0.04061333338419597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,12288,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,5120,0.015847466389338174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,8192,0.023010132710138957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,6144,0.018437333901723228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,7168,0.020312533775965372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,65536,0.1513535976409912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,4096,0.013529599706331889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,3584,0.012522666652997335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,3072,0.011432533462842304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,2560,0.010116266210873921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,2048,0.008370133241017659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,1536,0.007400533556938172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,256,0.005043200155099233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,1024,0.006346666812896728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,512,0.005487999816735586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,768,0.005797333518664042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,128,0.004664533336957296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,12288,0.028201599915822346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,16384,0.03741759856541951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,10240,0.02441493272781372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,8192,0.0211626668771108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,7168,0.01851946711540222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,65536,0.13546454111735026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,6144,0.016707199811935424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,2560,0.00909546713034312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,5120,0.014697600404421488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,4096,0.012775466839472453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,3584,0.011715199549992878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,3072,0.010615467031796774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,1024,0.006100266675154368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,768,0.005579733351866404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,2048,0.007698133091131846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,1536,0.00689386675755183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,16384,0.03784853219985962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,512,0.005377066632111868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,128,0.004660266637802124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,256,0.004869333406289419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,12288,0.028230400880177815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,10240,0.02472426692644755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,8192,0.021600000063578286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,6144,0.016775466998418174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,7168,0.018654932578404747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,5120,0.014872533082962037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,65536,0.1270016034444173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,4096,0.012330666184425354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,3072,0.010268800457318624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,3584,0.011477333307266236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,512,0.005339733262856802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,2560,0.009268266956011455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,2048,0.008162133395671844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,1536,0.007186133166154225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,1024,0.006257066627343495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,768,0.005818666517734527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,256,0.004955733319123586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,128,0.004698666433493296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,16384,0.0320362667242686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,12288,0.024171733856201173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,10240,0.021112533410390218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,8192,0.018917334079742432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,7168,0.015946666399637856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,5120,0.012553600470225015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,65536,0.10791040261586507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,6144,0.014469333489735923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,3072,0.00881599982579549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,2048,0.007234133283297221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,2560,0.007957333326339721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,4096,0.010551466544469198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,3584,0.009465600053469341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,768,0.0054954667886098225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,128,0.004665599763393402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,1536,0.006596266726652781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,1024,0.005817600091298421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,512,0.005178666611512502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,10240,0.017458132902781167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,256,0.004827733337879181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,16384,0.025363200902938844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,12288,0.019756799936294554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,8192,0.015059199929237366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,65536,0.08894933064778646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,7168,0.01337493360042572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,6144,0.011570133765538533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,5120,0.00993386705716451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,4096,0.008724266290664673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,3072,0.007562666634718577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,3584,0.008381866415341695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,768,0.00528106689453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,2560,0.007228800157705943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,1536,0.006011733412742614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,2048,0.006415999929110209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,1024,0.0053941334287325535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,512,0.004935466746489207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,256,0.004739200075467428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,16384,0.022021333376566567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,128,0.004555733501911163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,10240,0.015585066874821982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,12288,0.017591466506322227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,65536,0.07721066474914551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,8192,0.012759466965993246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,7168,0.011637333035469054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,6144,0.010539733370145162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,5120,0.00944213370482127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,4096,0.00839573343594869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,3584,0.008212266862392426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,3072,0.007378133138020833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,2560,0.007030400137106578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,2048,0.006366933385531108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,1536,0.005913599828879039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,512,0.004859733581542969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,1024,0.005369600156943003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,768,0.005074133475621542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,256,0.004673066735267639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,128,0.004488533238569895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,16384,0.02131519913673401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,12288,0.016781866550445557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,10240,0.014365866780281067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,8192,0.012264532844225566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,65536,0.07103253205617269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,2560,0.006960000097751618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,7168,0.011289599537849426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,6144,0.010244266192118327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,5120,0.009272533655166625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,4096,0.008270933230717977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,3584,0.00808426688114802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,3072,0.007282133400440216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,1536,0.005860266586144766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,2048,0.006260266900062561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,1024,0.0052373334765434265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,768,0.005070933202902476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,512,0.004833066463470459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,256,0.004574933151404063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,128,0.004491733511288961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,65536,0.07021973133087159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,16384,0.020667733748753865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,10240,0.014171733458836874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,12288,0.016075733304023742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,8192,0.012248532970746358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,7168,0.011294933160146077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,6144,0.010297600428263347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,5120,0.0093450665473938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,4096,0.00832533339659373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,3584,0.008030933141708375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,3072,0.007236266632874806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,2560,0.0069365332523981735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,2048,0.00624533345301946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,1536,0.005901866654555003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,1024,0.005179733534653981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,768,0.0050122668345769245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,512,0.004744533201058706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,256,0.004570666452248891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,128,0.00445119986931483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,65536,0.07062719662984213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,16384,0.020360533396402994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,12288,0.01606613298257192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,10240,0.014138666788736978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,8192,0.01225493351618449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,7168,0.011248000462849935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,6144,0.010237866640090942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,5120,0.009324799974759419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,4096,0.00827946662902832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,3584,0.008085333307584127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,3072,0.007269333302974701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,2560,0.007001600166161854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,2048,0.006146133442719777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,1536,0.005854933460553487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,1024,0.005305600166320801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,256,0.004587733248869578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,512,0.004724266628424326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,768,0.005064533154169718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,128,0.004418133199214936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,5120,0.1146122694015503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,6144,0.13594773610432942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,7168,0.156441593170166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,8192,0.17732799847920735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,10240,0.22034667332967123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,12288,0.26397973696390786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,4096,0.09341653188069662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,3072,0.07221120198567708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,3584,0.08304320176442465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,2560,0.061705601215362546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,2048,0.054586664835611975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,16384,0.34883092244466146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,1536,0.044436268011728924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,256,0.012378666798273723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,512,0.017449599504470826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,1024,0.029024000962575274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,768,0.023243733247121177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,128,0.009563733140627544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,6144,0.03874239921569824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,8192,0.049345068136850995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,7168,0.044021332263946535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,10240,0.06030186812082926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,12288,0.07097173531850179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,16384,0.09176533222198487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,4096,0.02781760096549988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,5120,0.03328853249549866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,2560,0.01970133384068807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,3584,0.02534719904263814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,3072,0.022402133544286093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,2048,0.016551466782887776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,256,0.005381333331267038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,512,0.007785599927107494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,768,0.009347200393676758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,128,0.005037866532802582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,1024,0.010712533195813497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,1536,0.0135861337184906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,7168,0.03578133185704549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,8192,0.039895466963450116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,10240,0.048344532648722335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,12288,0.057443201541900635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,16384,0.07402559916178385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,65536,0.34726505279541015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,5120,0.027379200855890913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,6144,0.031568000713984176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,4096,0.023082667589187623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,2560,0.018256000677744546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,3072,0.018776534001032512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,3584,0.021245867013931274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,2048,0.013916800419489542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,1024,0.009484799702962239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,1536,0.011720533172289532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,768,0.008301866551240284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,512,0.005838933090368906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,256,0.0052821333209673565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,128,0.004937600096066793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,65536,0.27577387491861977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,8192,0.03471680084864299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,10240,0.04172159830729167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,12288,0.04796479940414429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,16384,0.06193813482920328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,7168,0.030821333328882854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,6144,0.030877866347630817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,5120,0.02397226691246033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,4096,0.020558933417002358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,3584,0.018297600746154784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,3072,0.0164874662955602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,2560,0.014765866597493491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,2048,0.012753066420555115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,1536,0.010883200168609618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,768,0.007230933507283528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,1024,0.008840533097585042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,512,0.005646933118502299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,256,0.005283200244108836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,65536,0.23482240041097008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,128,0.0049098665515581764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,10240,0.033803733189900716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,12288,0.0397055983543396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,8192,0.02841599980990092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,16384,0.05003626743952433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,7168,0.02867199977238973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,6144,0.023053866624832154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,3072,0.013688533504803976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,3584,0.015683199961980185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,5120,0.020088533560434975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,4096,0.01716053287188212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,2048,0.010919466614723206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,2560,0.01258026659488678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,65536,0.18032639821370441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,512,0.005636266867319743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,768,0.006090666850407918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,1024,0.007868800063927968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,256,0.005165866514046987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,1536,0.00972266693909963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,128,0.00487253318230311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,7168,0.023105067014694215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,6144,0.020657066504160562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,10240,0.030598400036493938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,8192,0.0257205327351888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,12288,0.03590826590855916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,16384,0.045018664995829266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,4096,0.015390933553377787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,3584,0.014151466886202493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,3072,0.01272213359673818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,2560,0.011748266220092774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,5120,0.018150399128595986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,2048,0.010312533378601075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,65536,0.15897173881530763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,256,0.0051018665234247845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,128,0.004830933113892873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,512,0.005629866818586985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,1024,0.007046400010585785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,768,0.006055466830730438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,1536,0.00928106705347697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,6144,0.018782933553059898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,8192,0.022961066166559855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,10240,0.027474133173624675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,7168,0.020899200439453126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,12288,0.031946667035420734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,16384,0.039468801021575926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,5120,0.016309332847595216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,4096,0.013873066504796347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,3072,0.011814399560292562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,3584,0.013027200102806091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,2048,0.009590400258700053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,2560,0.010966400305430096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,65536,0.13877545992533366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,1024,0.006390400230884552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,512,0.00553173323472341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,256,0.005016533533732096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,768,0.005831466615200042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,128,0.004841599861780802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,1536,0.008778666456540424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,6144,0.016595199704170227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,10240,0.02430400053660075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,7168,0.018710400660832724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,12288,0.02821226716041565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,8192,0.020114133755366005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,16384,0.034884266058603924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,65536,0.12024426460266113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,5120,0.01455466647942861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,3072,0.010968533158302308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,4096,0.012616533041000366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,2048,0.008970666925112407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,3584,0.01204800009727478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,2560,0.010345600048700969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,1536,0.008462933699289958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,512,0.0053258667389551794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,256,0.005162666738033295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,768,0.005686399837334951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,1024,0.0059456000725428265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,128,0.004863999783992767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,6144,0.014924800395965577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,7168,0.016401066382726034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,16384,0.030877866347630817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,8192,0.01799359917640686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,10240,0.021577600638071695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,12288,0.024602667490641276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,65536,0.10532266298929852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,5120,0.013424000144004822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,2048,0.008184533317883809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,4096,0.011544533570607503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,3072,0.010203733046849569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,3584,0.011181867122650147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,2560,0.009641599655151368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,1536,0.00722453345855077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,512,0.005334400137265523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,768,0.005548800031344095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,256,0.005010133484999338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,1024,0.00591893345117569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,128,0.004819199939568837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,16384,0.029971200227737426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,8192,0.01755839983622233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,6144,0.014761599898338317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,7168,0.016250666975975037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,10240,0.0198527991771698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,12288,0.023050665855407715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,65536,0.10298879941304524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,5120,0.013209600249926248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,4096,0.011442133784294128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,3584,0.01097706655661265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,3072,0.010076799988746643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,2560,0.009134933352470398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,2048,0.007712000111738841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,1536,0.007179733117421467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,1024,0.006123733520507812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,768,0.005571199953556061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,256,0.005037866532802582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,512,0.005369600156943003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,128,0.004752000172932943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,16384,0.02794346610705058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,12288,0.022477867205937703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,10240,0.019569067160288493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,6144,0.013267200191815695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,8192,0.016669867436091106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,7168,0.015386666854222616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,65536,0.08908373514811198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,5120,0.012641066312789917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,4096,0.011037866274515789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,3584,0.0107424000898997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,3072,0.009352533022562663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,2560,0.00886079967021942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,2048,0.0074325333038965866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,1536,0.006705066561698914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,1024,0.005996799965699514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,512,0.005284266670544943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,768,0.005569066603978475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,256,0.004996266464392344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,128,0.004790399968624115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,12288,0.02199466625849406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,16384,0.025415466229120894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,10240,0.019248000780741372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,65536,0.09391040007273356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,8192,0.016404267152150473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,7168,0.014237866799036662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,6144,0.013792000214258828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,5120,0.012495999534924824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,4096,0.010442666212717692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,3072,0.009142399827639262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,3584,0.010469333330790202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,2048,0.007062399884064992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,1536,0.006735999882221222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,2560,0.008541867136955261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,1024,0.005990399916966756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,768,0.005504000186920166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,512,0.005291733145713806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,256,0.004999466737111409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,128,0.0046965335806210835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,12288,0.02037546634674072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,16384,0.02531733314196269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,10240,0.017563732465108235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,65536,0.08541119893391927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,8192,0.015478400389353433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,7168,0.014364799857139588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,6144,0.012794666488965354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,5120,0.011900800466537475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,4096,0.010496000448862713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,3584,0.009594666957855224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,2048,0.007039999961853028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,3072,0.00869546631971995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,2560,0.008056533336639405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,1024,0.005869866907596588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,1536,0.006557866434256236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,768,0.005644799768924713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,512,0.005234133203824361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,256,0.004972800115744273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,128,0.004769066472848257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,16384,0.024614399671554564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,12288,0.019827200969060262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,10240,0.017498666048049928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,65536,0.08236906528472901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,8192,0.015132799744606018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,7168,0.013948800166447959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,6144,0.01267413298288981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,5120,0.011410133043924967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,4096,0.009478400150934856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,3072,0.008137600123882293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,3584,0.009246933460235595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,2560,0.00784853349129359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,2048,0.006962133447329204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,1536,0.006691200037797292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,1024,0.005870933334032694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,768,0.005454933146635691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,512,0.005198933184146881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,256,0.004906666775544485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,128,0.004771199822425842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,12288,0.019826134045918785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,16384,0.024547199408213295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,65536,0.08214613596598307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,10240,0.017429333925247193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,8192,0.014761599898338317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,6144,0.012149332960446676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,7168,0.013333333532015481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,5120,0.010628267129262289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,4096,0.00936853289604187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,3072,0.008152533570925396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,3584,0.009155199925104777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,2560,0.007818666597207386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,2048,0.006922666728496551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,1536,0.006637866795063019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,1024,0.005876266459623972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,512,0.0052042668064435325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,256,0.004905599852403005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,768,0.005427200098832448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,128,0.004753066599369049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,16384,0.02446720004081726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,65536,0.08220907052357992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,12288,0.019662932554880778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,8192,0.014425599575042724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,7168,0.013195733229319254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,10240,0.017003732919692992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,6144,0.011986133456230164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,5120,0.010674132903416952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,4096,0.009354666868845622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,3584,0.009139200051625569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,3072,0.0081535999973615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,2048,0.006919466455777486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,2560,0.007845333218574524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,1536,0.006669866542021434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,1024,0.005874133110046387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,768,0.005378133555253347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,512,0.005211733281612396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,128,0.004643199841181437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,256,0.00489386667807897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,16384,0.02411839962005615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,65536,0.08203199704488119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,12288,0.01939520041147868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,10240,0.01687999963760376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,7168,0.012948266665140786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,8192,0.014261333147684732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,6144,0.01172693371772766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,5120,0.010630399982134501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,4096,0.009296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,3584,0.009179733196894328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,3072,0.008141866823037466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,2048,0.007003733515739441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,1536,0.006679466863473256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,2560,0.007783466577529907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,1024,0.005783466498057047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,768,0.005499733487764994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,256,0.004860800007979075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,512,0.005085866649945577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,128,0.004714666803677877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,65536,0.08165012995402018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,16384,0.0237119992574056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,12288,0.018809600671132406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,10240,0.01648853321870168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,8192,0.014145066340764364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,6144,0.011774933338165284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,7168,0.012958932916323343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,5120,0.010542933146158855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,4096,0.009290666381518046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,3584,0.00906773308912913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,3072,0.008108800152937572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,2048,0.006940799951553345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,2560,0.007727999985218048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,1536,0.00660693347454071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,1024,0.005774933099746704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,512,0.005111466844876607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,768,0.005436799923578898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,256,0.004836266736189524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,128,0.004670933385690053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,65536,0.08096319834391276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,16384,0.023565866549809775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,12288,0.01876373291015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,8192,0.014086400469144186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,10240,0.016429866353670754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,7168,0.012957866986592612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,6144,0.01179200013478597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,5120,0.010545066992441813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,4096,0.009338666995366413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,3584,0.009035733342170716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,3072,0.008070399860541026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,2560,0.007740800082683563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,2048,0.006865066786607106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,1536,0.006611200173695882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,1024,0.005815466741720835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,512,0.005074133475621542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,768,0.005334400137265523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,256,0.0048991998036702475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,128,0.004626133541266123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,5120,0.11305173238118489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,6144,0.13468480110168457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,7168,0.1550495942433675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,8192,0.17576212882995607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,10240,0.2187712033589681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,12288,0.262063995997111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,4096,0.09208959738413493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,3584,0.08184639612833658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,3072,0.07086186408996582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,2560,0.062226132551829016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,2048,0.05655573209126791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,512,0.017095466454823814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,1024,0.02929813265800476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,768,0.02507839997609456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,1536,0.043995734055836996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,16384,0.3437941233317057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,256,0.01221440037091573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,128,0.009332266449928284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,7168,0.04352853298187256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,8192,0.05655893484751383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,10240,0.05958720048268636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,6144,0.03845973412195842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,12288,0.07048959732055664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,16384,0.09112533728281656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,4096,0.02757866581281026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,3584,0.025145600239435833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,5120,0.032927999893824264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,2560,0.019569067160288493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,3072,0.022204800446828207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,2048,0.016359466314315795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,1024,0.010600533088048298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,768,0.009320533275604248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,512,0.007806933422883351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,256,0.005504000186920166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,1536,0.013516799608866373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,128,0.0050335998336474095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,10240,0.0479200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,8192,0.03957759936650594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,7168,0.03550080060958862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,12288,0.05555306673049927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,16384,0.07325332959493001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,6144,0.03126399914423625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,5120,0.027024000883102417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,4096,0.02290239930152893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,3584,0.020703999201456706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,3072,0.018506666024525963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,65536,0.343446413675944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,2560,0.016316800316174825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,2048,0.013622400164604188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,1024,0.009393067161242167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,1536,0.011742933591206869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,768,0.00824533353249232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,512,0.005804799993832906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,256,0.005313066641489664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,128,0.004946133494377137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,65536,0.27558720906575523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,8192,0.03472533226013184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,16384,0.06210346619288126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,10240,0.04098453521728516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,12288,0.048204799493153885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,7168,0.030474666754404706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,6144,0.03072426716486613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,3072,0.016340266664822897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,4096,0.020132267475128175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,5120,0.023856000105539957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,3584,0.018574933211008705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,2560,0.014503467082977294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,2048,0.012602667013804117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,1536,0.010795733332633973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,768,0.007132799923419952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,512,0.005688533186912537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,1024,0.008826667070388794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,256,0.005220266679922739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,128,0.004941866795221964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,65536,0.2263871987660726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,10240,0.03348160187403361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,12288,0.03914346694946289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,16384,0.049618132909138996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,8192,0.028191999594370527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,7168,0.025358933210372924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,6144,0.02518933415412903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,3072,0.013717333475748697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,3584,0.015422933300336204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,5120,0.01986133257548014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,4096,0.016986666123072307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,2560,0.012542933225631714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,2048,0.011025066177050274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,768,0.0061706667145093284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,512,0.005611733098824819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,1536,0.009742933511734008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,1024,0.007725866635640462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,256,0.005220266679922739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,128,0.004801066716512045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,65536,0.179477326075236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,7168,0.022824533780415854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,8192,0.02551146745681763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,10240,0.03020159999529521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,5120,0.01802026629447937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,12288,0.03524693250656128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,16384,0.044419201215108235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,6144,0.023056000471115112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,3072,0.012531200051307678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,4096,0.015338666240374246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,3584,0.014061866203943887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,2560,0.011727999647458394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,2048,0.010263466835021972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,1536,0.009175466497739156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,1024,0.006532266736030579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,768,0.005995733539263407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,65536,0.15807894070943196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,512,0.005514666438102722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,256,0.005163733164469401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,128,0.004830933113892873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,16384,0.039228800932566324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,8192,0.02258346676826477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,7168,0.02068693240483602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,10240,0.028676267464955645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,6144,0.018583466609319053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,12288,0.03163946668306987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,5120,0.016123732924461363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,3072,0.011636267105738323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,4096,0.013898666699727377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,3584,0.012957866986592612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,2560,0.010942932963371278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,2048,0.009635200103123982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,65536,0.13804480234781902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,1536,0.00867733359336853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,1024,0.006472533444563548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,768,0.005713066458702088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,512,0.005490133166313171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,256,0.0050570666790008545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,128,0.004824533561865489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,16384,0.034409598509470625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,12288,0.027898667256037395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,6144,0.016679465770721436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,10240,0.02380266586939494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,7168,0.018425599733988444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,8192,0.020102399587631225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,2560,0.010225066542625427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,5120,0.014593066771825156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,4096,0.012660266955693564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,3584,0.01193173329035441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,3072,0.010880000392595927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,2048,0.008921600381533305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,65536,0.11894720395406086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,1536,0.007590400179227193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,128,0.004806399842103322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,1024,0.005851733187834421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,768,0.005825066566467285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,512,0.00547733356555303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,256,0.0050005331635475155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,10240,0.020883200565973918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,12288,0.02445759971936544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,16384,0.030403200785319013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,7168,0.016525866587956746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,8192,0.01807360053062439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,6144,0.014664533734321594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,5120,0.01304639975229899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,4096,0.011639466881752015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,2560,0.00963200032711029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,3072,0.01011306643486023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,3584,0.011013333002726238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,2048,0.008155733346939087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,65536,0.10409066677093506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,1024,0.005996799965699514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,1536,0.006837333242098491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,768,0.0056618665655454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,512,0.005373866856098175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,256,0.005032533407211303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,128,0.004790399968624115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,16384,0.02970133423805237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,12288,0.022636799017588298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,8192,0.017500799894332886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,6144,0.01388053297996521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,7168,0.016033066312472026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,10240,0.020884267489115396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,5120,0.013048533598581949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,4096,0.011015466849009196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,65536,0.10123519897460938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,2048,0.007804800073305767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,3584,0.011032533645629884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,3072,0.009648000200589497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,2560,0.009355733791987102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,512,0.005374933282534281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,768,0.0056874667604764305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,1536,0.006830933193365733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,1024,0.005850666761398315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,256,0.005053866902987162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,128,0.004713599880536398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,10240,0.019433599710464478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,16384,0.027893332640329997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,12288,0.022194133202234904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,8192,0.015493333339691162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,7168,0.015271466970443726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,6144,0.0131221334139506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,3584,0.010506666700045268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,3072,0.009305600325266521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,65536,0.09379733403523763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,5120,0.01245013376077016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,4096,0.011025066177050274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,768,0.00555626650651296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,2048,0.007039999961853028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,2560,0.008643200000127155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,1536,0.0067104001839955645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,1024,0.005834666887919108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,512,0.005301333467165629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,256,0.004936533172925314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,128,0.004716800153255462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,10240,0.017658666769663493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,12288,0.021811199188232423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,16384,0.027410133679707842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,7168,0.015044266978899637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,8192,0.016238933801651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,3072,0.009046399593353271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,6144,0.013702399532000222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,5120,0.011749333143234253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,65536,0.08360746701558432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,3584,0.010401067137718201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,4096,0.010806399583816528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,2560,0.008458667000134786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,768,0.005438933273156484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,256,0.004962133367856344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,2048,0.00727893312772115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,1536,0.006749866902828217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,1024,0.005845333139101664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,512,0.005298133194446564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,128,0.004744533201058706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,12288,0.020267732938130698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,16384,0.02514773408571879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,8192,0.014946132898330688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,10240,0.017845332622528076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,7168,0.014251733819643656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,65536,0.08228800296783448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,6144,0.013012267152468362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,5120,0.011717333396275838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,3584,0.009397332866986591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,4096,0.010296533505121868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,3072,0.008761599659919739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,2048,0.00689386675755183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,2560,0.007971199850241344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,1536,0.00653653343518575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,1024,0.005843199789524078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,768,0.005446400245030721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,512,0.005236266553401947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,256,0.004894933104515076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,128,0.004692266881465912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,16384,0.024392533302307128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,12288,0.019617066780726115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,10240,0.01727893352508545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,65536,0.0818399985631307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,8192,0.014896000425020853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,7168,0.013796266913414002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,6144,0.01249066690603892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,5120,0.01118293305238088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,4096,0.009530666470527648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,3584,0.009118933478991191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,1536,0.006593066453933716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,3072,0.008183466891447704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,2560,0.00780266672372818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,2048,0.0068906664848327635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,1024,0.005783466498057047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,768,0.005411200225353241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,512,0.005165866514046987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,256,0.00489386667807897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,128,0.00472320020198822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,65536,0.08156159718831381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,16384,0.02422399918238322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,12288,0.01951040029525757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,10240,0.017199999094009398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,8192,0.014734933773676554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,7168,0.013290666540463767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,6144,0.01183893382549286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,5120,0.010453333457310993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,4096,0.009297066926956176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,2048,0.006925866504510244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,3584,0.00906880001227061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,3072,0.00808426688114802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,2560,0.007772799829641979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,1536,0.006664533416430156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,128,0.0046741331617037455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,1024,0.005769599974155426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,512,0.005126399795214335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,768,0.00537066658337911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,256,0.004861866434415182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,65536,0.08144533634185791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,12288,0.019182932376861573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,16384,0.024196267127990723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,10240,0.016852267583211265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,8192,0.014406399925549826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,7168,0.012966400384902954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,6144,0.011611732840538024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,3584,0.00906986693541209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,3072,0.008081066608428954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,5120,0.010475732882817586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,4096,0.00921493371327718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,2560,0.007768533130486806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,2048,0.006962133447329204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,1536,0.006562133133411407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,1024,0.005760000149408976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,768,0.005367466807365417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,512,0.005134933193524678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,256,0.0047775998711586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,128,0.0045855998992919925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,16384,0.023803732792536416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,65536,0.08126933574676513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,12288,0.01920959949493408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,10240,0.016451199849446617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,8192,0.01394773324330648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,7168,0.01297920048236847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,6144,0.01154026687145233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,2560,0.007778133451938629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,5120,0.010405332843462626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,4096,0.009212799866994222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,3584,0.009058133761088053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,3072,0.008077866832415263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,768,0.0053375999132792154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,2048,0.006874666611353557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,512,0.005098666747411092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,1536,0.0065653334061304735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,1024,0.005773866673310598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,256,0.004833066463470459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,128,0.0045973335703214015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,65536,0.08117547035217285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,16384,0.023449599742889404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,12288,0.018659200270970663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,10240,0.016250666975975037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,8192,0.013884799679120383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,7168,0.012824533383051553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,4096,0.009246933460235595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,6144,0.011564800143241882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,5120,0.010414933164914448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,3584,0.008983467022577922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,2560,0.007669333120187123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,3072,0.008054399987061818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,2048,0.006964266796906789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,1024,0.005684266487757364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,1536,0.006558933357397716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,768,0.0053375999132792154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,512,0.005047466854254405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,256,0.004781866570313772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,128,0.004572799801826477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,16384,0.02331519921620687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,65536,0.08059306939442953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,12288,0.018616533279418944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,10240,0.016251732905705772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,7168,0.012737066547075907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,8192,0.01393066644668579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,6144,0.011517866452534994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,5120,0.010382933417956035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,4096,0.009231999516487122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,3584,0.008933333555857341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,3072,0.008027733365694682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,2560,0.007682133217652638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,2048,0.006843733290831248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,1536,0.006526933113733928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,1024,0.005700266857941946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,768,0.005371733506520589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,512,0.005027199784914652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,256,0.004710400104522705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,128,0.004632533093293508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,5120,0.11267626285552979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,6144,0.1339765389760335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,7168,0.15469973882039387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,8192,0.17477973302205402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,10240,0.21821653048197426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,12288,0.26075305938720705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,4096,0.09126826922098795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,3072,0.07030719916025797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,3584,0.08108373483022055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,2560,0.06165866851806641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,1536,0.04398186604181926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,128,0.0092031995455424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,2048,0.05651413202285767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,1024,0.029173332452774047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,256,0.012190933028856914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,768,0.02262186606725057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,512,0.01858773430188497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,16384,0.3455850601196289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,7168,0.04326293468475342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,10240,0.05932693481445313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,12288,0.0702293316523234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,8192,0.048529068628946945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,6144,0.04379946788152059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,16384,0.09079999923706054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,4096,0.02733653386433919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,5120,0.03264533281326294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,3584,0.02481279969215393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,3072,0.021919999519983926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,2560,0.019449599583943687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,2048,0.01616213321685791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,256,0.005569066603978475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,1536,0.013484799861907959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,768,0.009204266468683879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,512,0.007739733159542084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,1024,0.010539733370145162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,128,0.00499839981396993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,8192,0.03942293326059977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,12288,0.05547413428624472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,7168,0.035197865962982175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,10240,0.048002131779988605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,16384,0.07405760288238525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,3584,0.02090346614519755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,6144,0.031128533681233722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,4096,0.02253440022468567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,5120,0.02653653422991435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,1024,0.009302399555842082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,1536,0.011686399579048157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,2048,0.013748266299565635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,3072,0.018513067563374837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,768,0.008249600231647492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,2560,0.016489600141843162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,65536,0.3432469367980957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,128,0.004916266600290934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,512,0.005907199780146281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,256,0.005312000215053558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,16384,0.06096320152282715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,7168,0.030261333783467608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,10240,0.04140479962031047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,12288,0.05586346785227457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,8192,0.033613868554433185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,65536,0.2785717328389486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,3072,0.016403200229008992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,6144,0.02720213333765666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,5120,0.023628799120585124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,4096,0.020230400562286376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,3584,0.018363734086354576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,1536,0.010840533177057902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,2560,0.01606933375199636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,1024,0.008737066388130188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,2048,0.01253973344961802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,128,0.004794666667779287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,768,0.0070783997575442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,512,0.005782400071620941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,256,0.00520000010728836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,65536,0.22734400431315102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,12288,0.03918079932530721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,8192,0.027837866544723512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,10240,0.0333898663520813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,7168,0.025183999538421632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,6144,0.02521386742591858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,16384,0.05016746520996094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,3584,0.015520000457763672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,5120,0.02022613286972046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,2560,0.012474667032559712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,3072,0.013814399639765421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,2048,0.010883200168609618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,4096,0.016478932897249856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,1536,0.009897599617640179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,128,0.004785066843032837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,1024,0.007755733529726664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,768,0.005936000247796377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,512,0.0056991999348004665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,256,0.005143466591835022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,65536,0.1798869291941325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,16384,0.04412586688995361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,8192,0.025170133511225386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,10240,0.02986239989598592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,7168,0.022734934091567995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,12288,0.0352021336555481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,6144,0.023035732905069987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,4096,0.015309866269429526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,2048,0.01013973355293274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,3584,0.014057599504788718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,3072,0.01260693371295929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,5120,0.017848533391952515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,2560,0.011878400047620138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,512,0.005657599866390228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,128,0.004796800017356872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,1536,0.009304533402125042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,1024,0.006648533542950948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,768,0.005749333401521047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,65536,0.15732266108194987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,256,0.0051146666208903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,6144,0.018425599733988444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,8192,0.022383999824523926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,7168,0.023100799322128295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,5120,0.01597866714000702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,12288,0.031310933828353885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,10240,0.02690773407618205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,16384,0.0392138679822286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,4096,0.013847466309865317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,3584,0.013005866607030233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,2560,0.010969600081443787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,2048,0.00956053336461385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,3072,0.011621333161989848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,1536,0.008755200107892354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,65536,0.1368384043375651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,1024,0.006017066538333893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,768,0.00600853314002355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,512,0.005497600138187409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,128,0.004749866823355356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,256,0.004999466737111409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,16384,0.03435200055440267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,6144,0.016520532965660095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,8192,0.020038400093714395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,10240,0.023973333835601806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,7168,0.018230400482813516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,12288,0.02762346665064494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,5120,0.014403200149536133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,3584,0.0120469331741333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,4096,0.012659200032552085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,3072,0.010889599720637005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,2048,0.00888213316599528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,2560,0.01007466713587443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,65536,0.11794666449228923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,512,0.005515733361244201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,1024,0.006153599917888641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,1536,0.008003200093905132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,768,0.005846400062243144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,256,0.004904533425966898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,128,0.00472320020198822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,6144,0.014549332857131957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,16384,0.030456533034642536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,7168,0.016131200393040977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,12288,0.024572799603144325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,8192,0.018051199118296304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,10240,0.021401600042978922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,65536,0.1047594706217448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,5120,0.013019733627637229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,4096,0.011769599715868632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,3584,0.011025066177050274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,1536,0.0072543998559316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,3072,0.010188800096511842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,2560,0.009648000200589497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,2048,0.008214400211970011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,128,0.004677333434422811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,256,0.004946133494377137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,1024,0.005805866420269012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,768,0.005721599857012431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,512,0.005154133339722951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,8192,0.016410666704177856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,7168,0.016184533635775246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,16384,0.029778132836023968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,12288,0.024075732628504435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,10240,0.02105600039164225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,6144,0.013738666971524557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,2560,0.009452799956003826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,4096,0.011681066950162251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,3072,0.010005333026250203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,5120,0.012429866194725036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,3584,0.011136000355084736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,65536,0.10141226450602214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,2048,0.007357866565386455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,1536,0.007166933516661327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,1024,0.0060928001999855045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,768,0.0056981335083643595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,512,0.005251200000445048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,256,0.00498879998922348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,128,0.004673066735267639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,16384,0.027929600079854327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,10240,0.0197269340356191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,8192,0.01548693378766378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,6144,0.01394773324330648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,12288,0.022514132658640544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,7168,0.01530346671740214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,4096,0.011055999994277954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,65536,0.08672213554382324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,2560,0.008984532952308655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,5120,0.012532266974449157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,1024,0.005817600091298421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,3584,0.010263466835021972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,3072,0.009443199634552002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,2048,0.007433600227038066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,256,0.004796800017356872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,1536,0.006937600175539653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,768,0.005673600236574808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,512,0.005262933174769084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,128,0.004668800036112467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,10240,0.019486933946609497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,12288,0.02007466753323873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,16384,0.027371732393900554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,8192,0.01639893352985382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,7168,0.015159466862678527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,3584,0.010544000069300334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,6144,0.012837333480517068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,5120,0.01253546675046285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,65536,0.09233813285827637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,2560,0.008875733613967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,4096,0.01032426655292511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,3072,0.009481599926948548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,2048,0.007409066458543142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,1536,0.006607999900976817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,1024,0.005901866654555003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,768,0.005580799778302511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,256,0.004795733094215393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,512,0.00528959979613622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,128,0.004702933132648468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,12288,0.02057386636734009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,8192,0.014968533317248026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,5120,0.011797333757082622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,10240,0.017863466342290243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,16384,0.024504532416661583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,6144,0.013089066743850708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,7168,0.014409599701563516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,3072,0.008834133545557659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,4096,0.010062932968139648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,65536,0.08562453587849936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,3584,0.009799466530481974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,768,0.005345066885153452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,512,0.005242666602134705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,2048,0.0071946665644645694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,2560,0.007765333354473114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,1536,0.00660693347454071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,1024,0.005809066692988077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,16384,0.024244266748428344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,10240,0.017230933904647826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,256,0.004885333279768625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,128,0.004557866851488749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,12288,0.01950826644897461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,4096,0.009504000345865887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,8192,0.014824533462524414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,7168,0.013688533504803976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,5120,0.011101866761843365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,6144,0.012453333536783854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,65536,0.08133440017700196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,3584,0.009085866808891296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,3072,0.008097066481908163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,2560,0.0077248002092043565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,2048,0.006900266806284587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,1536,0.0065738668044408154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,1024,0.005737600227197012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,768,0.005328000088532766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,512,0.005098666747411092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,256,0.004755199948946635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,128,0.004647466540336609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,16384,0.02409279942512512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,12288,0.01941653291384379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,10240,0.017076265811920167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,65536,0.08113919893900554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,8192,0.01456000010172526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,4096,0.009288533528645834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,3584,0.00904746651649475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,7168,0.013541332880655923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,6144,0.011705600221951802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,5120,0.010614400108655293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,3072,0.008045866588751475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,2560,0.007674666742483775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,768,0.005346133311589559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,1536,0.006534400085608165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,2048,0.006870399912198384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,1024,0.005764266848564148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,512,0.005089066425959269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,256,0.004824533561865489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,10240,0.016541866461435954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,128,0.004604800045490265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,65536,0.08104533354441325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,8192,0.014382933576901754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,16384,0.024062933524449666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,12288,0.019322667519251505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,7168,0.012984533111254373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,6144,0.011577600240707397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,3072,0.008085333307584127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,5120,0.010455466310183207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,4096,0.009308800101280212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,3584,0.009063466389973959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,768,0.005306666592756907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,2560,0.007773866752783458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,2048,0.006899199883143107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,128,0.004538666705290476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,256,0.004735999802748362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,1536,0.006583466629187266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,1024,0.005709866682688395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,65536,0.08077759742736816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,8192,0.013933866222699483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,512,0.005077333251635233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,16384,0.023779199520746867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,12288,0.019036799669265747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,10240,0.016570666432380678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,4096,0.009218133489290873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,7168,0.012756266196568809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,3584,0.009046399593353271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,3072,0.008021333316961924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,6144,0.011614933609962463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,5120,0.010386133193969726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,1536,0.006516266862551372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,2560,0.0076906666159629825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,2048,0.006865066786607106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,512,0.004995200037956238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,128,0.004570666452248891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,1024,0.005756799876689911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,768,0.005297066768010458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,12288,0.018569600582122803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,256,0.004731733103593191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,65536,0.08056000073750814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,16384,0.02327359914779663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,10240,0.016198399662971496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,8192,0.0139082670211792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,4096,0.009177600344022114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,7168,0.012708266576131185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,6144,0.011573333541552227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,5120,0.010392533739407857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,768,0.0052714665730794275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,3584,0.008932266632715862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,3072,0.007979733248551685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,2048,0.006844800213972728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,2560,0.00768746683994929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,1536,0.0065087998906771345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,1024,0.005659733215967814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,256,0.0046741331617037455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,512,0.005003733436266581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,128,0.004525866607824961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,16384,0.02326293389002482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,7168,0.012710400422414145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,65536,0.07986346880594888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,12288,0.018480000893274943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,10240,0.016189866264661155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,8192,0.013867732882499696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,3584,0.008920533458391826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,2048,0.006805333495140076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,6144,0.01151146690050761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,5120,0.010396800438563029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,4096,0.009172266721725464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,3072,0.007993599772453308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,2560,0.007667199770609538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,1536,0.006602666775385539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,256,0.004681600133577982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,1024,0.005633066594600678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,512,0.0050016000866889955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,768,0.005182933310667674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,128,0.004513066510359446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,5120,0.11185173193613689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,6144,0.1328330675760905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,7168,0.15392106374104816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,8192,0.1741162618001302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,10240,0.21887466112772622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,12288,0.260807466506958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,4096,0.09074453512827554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,3072,0.06987093289693197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,3584,0.08067413171132407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,2560,0.06093759934107462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,2048,0.05634239912033081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,1024,0.029040000836054486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,16384,0.34318507512410484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,1536,0.04363946517308553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,768,0.022367999951044718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,512,0.018410666783650716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,256,0.012054399649302164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,128,0.009179733196894328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,7168,0.043049601713816325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,8192,0.04833386739095052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,12288,0.0702069362004598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,10240,0.05910826524098715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,6144,0.037913600603739425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,2560,0.01932800014813741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,4096,0.027124265829722088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,16384,0.10627199808756511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,5120,0.03246399958928426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,3072,0.021949867407480873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,3584,0.024701867500940958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,1536,0.013384532928466798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,128,0.004859733581542969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,768,0.00921493371327718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,1024,0.01053653359413147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,256,0.005545599758625031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,512,0.007612800101439159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,2048,0.016225066781044007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,8192,0.03928639888763428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,10240,0.04789973497390747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,16384,0.07405440012613931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,12288,0.05667946736017863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,7168,0.0347978671391805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,5120,0.02672213315963745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,6144,0.030876799424489336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,2560,0.01629973351955414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,4096,0.02243946592013041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,3584,0.020574933290481566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,3072,0.018219733238220216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,2048,0.013649066289265951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,1536,0.011661866307258606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,65536,0.34212799072265626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,1024,0.009217066566149394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,768,0.008252800007661184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,128,0.004779733220736186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,512,0.0058229332168896995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,256,0.0052928000688552855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,7168,0.029958399136861165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,8192,0.03423999945322673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,10240,0.04166719913482666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,12288,0.04815680185953776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,65536,0.27333974838256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,6144,0.027138133843739826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,16384,0.06273279984792074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,4096,0.01999893387158712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,2560,0.014524799585342408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,3072,0.0181386669476827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,3584,0.018360533316930137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,1536,0.010775466759999592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,5120,0.022895999749501548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,512,0.005738666653633118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,256,0.005121066669623057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,2048,0.012536533673604331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,1024,0.008779733379681905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,768,0.007187200089295705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,128,0.004763733347256978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,12288,0.03884586493174235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,65536,0.2313162644704183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,10240,0.033352533976236984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,8192,0.027829333146413164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,16384,0.04981866677602132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,7168,0.024858667453130086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,3072,0.013768532872200012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,2560,0.012451199690500896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,6144,0.022538665930430093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,5120,0.021972266832987468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,4096,0.017061332861582436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,1536,0.00965226689974467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,3584,0.01551253298918406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,2048,0.011170132954915365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,1024,0.007698133091131846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,768,0.006050133208433787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,512,0.005662933488686879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,256,0.004993066688378652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,128,0.004755199948946635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,65536,0.1792319933573405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,7168,0.02257279952367147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,10240,0.030011733373006184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,16384,0.04385173320770264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,8192,0.025065600872039795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,12288,0.03501226504643758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,4096,0.015343999862670899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,3072,0.012625066439310708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,3584,0.013979732990264893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,6144,0.022887466351191203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,5120,0.017850667238235474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,2560,0.011556266744931539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,1024,0.006503466765085857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,512,0.005580799778302511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,1536,0.00913706620534261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,768,0.006153599917888641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,2048,0.010140800476074218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,256,0.005107200145721436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,128,0.004731733103593191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,65536,0.15657386779785157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,8192,0.022323199113210044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,7168,0.020361600319544475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,16384,0.03876800139745076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,5120,0.016006400187810264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,6144,0.02059626579284668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,10240,0.02666880091031392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,12288,0.031318400303522745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,3072,0.011601066589355469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,2560,0.010942932963371278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,4096,0.013839999834696451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,1536,0.008707200487454731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,2048,0.009525332848230999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,3584,0.01286720037460327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,512,0.005426133175690969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,128,0.004758400221665701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,256,0.005005866785844167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,1024,0.00614933321873347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,768,0.005952000121275584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,65536,0.13642133076985677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,12288,0.02751893401145935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,10240,0.023695999383926393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,7168,0.018198400735855103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,8192,0.01994026700655619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,16384,0.034125868479410806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,6144,0.018248534202575682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,3584,0.011966933806737263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,3072,0.010774399836858113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,1536,0.007957333326339721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,5120,0.01440000037352244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,4096,0.012633599837621055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,1024,0.005957333246866862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,2560,0.010154666503270467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,2048,0.008782933155695598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,768,0.005805866420269012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,65536,0.11724906762440998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,512,0.005465599894523621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,128,0.004679466784000397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,256,0.004986666639645894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,7168,0.01617173353830973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,12288,0.024524799982706704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,10240,0.02119999925295512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,5120,0.012919466694196066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,16384,0.030118399858474733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,8192,0.018626133600870766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,3584,0.011011200149854024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,6144,0.01503679951032003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,1536,0.006674133241176605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,4096,0.011673600474993388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,3072,0.010072533289591472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,768,0.005347200234731038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,2048,0.008534399668375652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,2560,0.009590400258700053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,65536,0.10418240229288737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,1024,0.006061866879463196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,512,0.005248000224431356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,256,0.004874666531880697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,128,0.004614399870236715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,16384,0.02963520089785258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,10240,0.020856533447901407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,6144,0.01383786698182424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,8192,0.017668267091115318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,12288,0.02401813268661499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,3072,0.010046933094660442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,7168,0.015029333035151162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,5120,0.013184000054995218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,65536,0.10252906481424968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,4096,0.011569066842397054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,2048,0.007899733384450276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,3584,0.010531199971834819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,2560,0.009372799595197042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,128,0.004580266773700714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,1536,0.006753066678841908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,1024,0.00598826656738917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,768,0.005756799876689911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,256,0.004889599978923798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,512,0.005297066768010458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,7168,0.015203199783960977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,10240,0.019486933946609497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,16384,0.02563520073890686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,8192,0.017014400164286295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,12288,0.0210207998752594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,6144,0.0139765332142512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,5120,0.012590932846069335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,4096,0.010986666878064473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,1536,0.006790400048096975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,3072,0.009726933638254802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,65536,0.09448213577270508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,3584,0.010262399911880493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,2560,0.008328533172607422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,2048,0.007449600100517273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,1024,0.005842133363087972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,768,0.005565866827964783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,256,0.004902400076389313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,512,0.005243733525276184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,128,0.004558933277924856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,12288,0.021978666385014854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,10240,0.019242666165033975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,8192,0.016718933979670204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,16384,0.027607466777165728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,65536,0.08240426381429036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,7168,0.015158399939537048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,6144,0.01279146671295166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,5120,0.012442666292190551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,4096,0.011014399925867717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,3584,0.010319999853769938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,3072,0.009337600072224934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,2560,0.00790719985961914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,1536,0.006554666658242543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,768,0.005569066603978475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,2048,0.007398400207360585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,1024,0.005853866537412008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,12288,0.020537600914637247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,256,0.004891733328501383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,128,0.004635733366012573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,512,0.005188266436258951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,10240,0.01790613333384196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,7168,0.014313600460688271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,16384,0.025300266345342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,8192,0.015619200468063355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,4096,0.010356266299883525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,6144,0.012544000148773193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,5120,0.011665067076683045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,65536,0.08160106341044107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,2560,0.008044800162315369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,2048,0.007088000078996022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,3072,0.008570667107899983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,3584,0.009690666198730468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,1024,0.005760000149408976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,768,0.005303466816743215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,256,0.004835199813048045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,1536,0.006438399851322174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,16384,0.02421226700146993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,12288,0.01954879959424337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,512,0.0052373334765434265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,128,0.004632533093293508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,8192,0.014769066373507181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,5120,0.011195733149846395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,3584,0.009089066584904989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,4096,0.00931946635246277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,10240,0.017146666844685875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,7168,0.013771733641624451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,3072,0.008081066608428954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,6144,0.012452266613642375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,65536,0.08120319843292237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,2560,0.007763200004895528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,1024,0.005718400080998739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,128,0.004631466666857402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,512,0.004995200037956238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,256,0.0047989333669344585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,2048,0.006868266562620799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,1536,0.006559999783833821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,768,0.005340800185998281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,8192,0.014255999525388082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,6144,0.011850666999816895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,5120,0.010417067011197408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,16384,0.024081067244211832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,12288,0.019372800985972084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,10240,0.017083734273910522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,7168,0.01306773324807485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,65536,0.08100693225860596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,2560,0.007768533130486806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,2048,0.006859733164310456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,4096,0.009256533781687419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,1536,0.006494933366775512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,3584,0.008986666798591614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,3072,0.008074666559696197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,256,0.004819199939568837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,768,0.005246933301289876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,1024,0.005712000032265982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,512,0.0050335998336474095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,12288,0.019273600975672402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,128,0.00461760014295578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,10240,0.016740266482035318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,7168,0.012787200013796487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,16384,0.023983999093373617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,65536,0.08094399770100912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,8192,0.01423679987589518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,6144,0.01158186693986257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,4096,0.009315199653307597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,5120,0.010335999727249145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,3072,0.00798933357000351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,3584,0.009057066837946574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,768,0.005277866621812185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,2560,0.007719466586907704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,2048,0.006798933446407318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,1536,0.006558933357397716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,1024,0.005637333293755849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,12288,0.019030400117238364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,16384,0.02364906668663025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,512,0.0049781332413355505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,256,0.004711466530958811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,128,0.004579199850559235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,8192,0.013978667060534158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,6144,0.011507200201352437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,65536,0.08079040050506592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,3584,0.009036800265312195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,10240,0.016335999965667723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,3072,0.008004266520341237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,2560,0.007708799839019775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,7168,0.012735999623934426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,2048,0.006888533135255177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,5120,0.01037440001964569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,4096,0.009161600470542907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,256,0.004725333551565806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,1536,0.00658133327960968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,1024,0.005676800012588501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,65536,0.08032533327738443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,768,0.0052042668064435325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,512,0.004947199920813243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,10240,0.016140799721082053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,128,0.004546133180459341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,16384,0.023246934016545616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,12288,0.01845759948094686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,8192,0.0138154665629069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,7168,0.012713600198427835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,6144,0.011480533083279927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,5120,0.010312533378601075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,4096,0.009144533673922222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,2048,0.006786133348941803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,3584,0.008917333682378133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,512,0.00489279975493749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,3072,0.008005333443482716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,2560,0.0075989335775375364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,1536,0.006454400221506755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,1024,0.005613866448402405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,16384,0.023142399390538533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,256,0.004668800036112467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,768,0.005194666484991709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,128,0.004474666714668274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,65536,0.08003520170847575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,12288,0.0184608002503713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,4096,0.009177600344022114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,10240,0.016105600198109946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,8192,0.01383039951324463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,7168,0.012770133217175803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,6144,0.011543466647466024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,3072,0.007957333326339721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,1536,0.006563200056552887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,5120,0.01034773290157318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,512,0.004983466863632202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,2560,0.007628799974918365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,3584,0.008836266398429871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,2048,0.006763733426729838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,1024,0.005590400099754334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,768,0.005173333485921224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,256,0.00462719996770223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,128,0.0045162667830785114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,5120,0.11112639904022217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,6144,0.1327071984608968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,7168,0.15551039377848308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,8192,0.17342294057210286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,10240,0.21756800015767416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,12288,0.2621493339538574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,2048,0.056141865253448484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,3072,0.06941546599070231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,3584,0.08026133378346761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,4096,0.09042239983876546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,2560,0.06063679854075114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,256,0.012227200468381246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,768,0.02216106653213501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,1024,0.027445334196090698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,512,0.01830186645189921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,128,0.00925973355770111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,1536,0.04346346855163574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,16384,0.34269545873006185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,6144,0.043884801864624026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,7168,0.04289493163426717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,12288,0.06997653643290201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,8192,0.048146132628122965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,10240,0.059290667374928795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,16384,0.09077333609263102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,4096,0.027013333638509114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,2048,0.016156799594561257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,3072,0.021702400843302407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,2560,0.019154133399327596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,5120,0.032264532645543416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,1536,0.013372799754142762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,3584,0.02450773318608602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,256,0.0053845331072807315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,512,0.007516799867153168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,1024,0.010442666212717692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,768,0.009215999643007915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,128,0.004911999901135763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,12288,0.05615573326746622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,8192,0.03874133427937825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,10240,0.04673920075098674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,7168,0.034782934188842776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,16384,0.07314453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,6144,0.0309663991133372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,2560,0.016124799847602844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,4096,0.022107734282811483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,3584,0.020466132958730062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,3072,0.01949013272921244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,5120,0.026706133286158246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,1024,0.009187199672063192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,65536,0.34216534296671547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,512,0.005885866781075796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,2048,0.013618133465449014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,1536,0.011487999558448791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,256,0.00516480008761088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,768,0.008261333405971526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,128,0.004890666902065277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,7168,0.029999999205271403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,8192,0.03418879906336467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,12288,0.0481824000676473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,10240,0.04152426719665527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,65536,0.27115306854248045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,16384,0.06297386487325032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,5120,0.02347946763038635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,6144,0.02674986720085144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,3072,0.017986132701237997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,3584,0.018071466684341432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,4096,0.019738666216532388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,2560,0.014338133732477823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,2048,0.012429866194725036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,1536,0.01088853379090627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,1024,0.008613333106040955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,256,0.005094400048255921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,768,0.007186133166154225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,512,0.0056320001681645715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,128,0.004792533318201701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,16384,0.05049813191095988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,7168,0.024851200977961223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,12288,0.038837333520253495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,8192,0.028246400753657024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,10240,0.033662935098012284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,65536,0.229477326075236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,5120,0.019382399320602418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,3072,0.013563733299573263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,6144,0.022232532501220703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,3584,0.01527466674645742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,1536,0.009608532985051472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,4096,0.016694400707880655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,768,0.005922133227189382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,2048,0.010883200168609618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,2560,0.013780267039934794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,512,0.005553066730499268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,1024,0.007573333382606506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,128,0.004678399860858917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,256,0.005051733553409576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,65536,0.17877972920735677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,8192,0.024820266167322795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,12288,0.03495573202768962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,16384,0.04370666742324829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,10240,0.02993280092875163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,5120,0.01767680048942566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,7168,0.02251840035120646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,3072,0.012555733323097229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,4096,0.015101866920789084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,6144,0.022951465845108033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,3584,0.014028799533843995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,1536,0.009082667032877604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,2560,0.011437867085138958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,1024,0.006614399949709575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,768,0.006089599927266439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,2048,0.010131200154622395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,256,0.0050784001747767125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,512,0.00557226687669754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,128,0.0047199999292691554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,12288,0.03101973334948222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,65536,0.15622933705647785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,10240,0.02667733430862427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,16384,0.03866666555404663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,5120,0.017924267053604125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,6144,0.018242132663726807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,8192,0.022396800915400188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,3584,0.0129120002190272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,3072,0.011574400464693706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,7168,0.020252799987792967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,2560,0.010878933469454448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,4096,0.013662933309872945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,1024,0.00610453337430954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,512,0.005420800050099691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,2048,0.009479467074076335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,128,0.004717866579691568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,768,0.005508266886075338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,1536,0.008550399541854858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,256,0.004951466619968414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,65536,0.13645866711934407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,10240,0.0234826664129893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,6144,0.018309332927068076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,5120,0.014446933070818582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,12288,0.02744106650352478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,8192,0.019990400473276774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,7168,0.018219733238220216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,16384,0.0341813325881958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,4096,0.012552533547083536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,3072,0.0107722669839859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,2048,0.008758399883906047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,1536,0.007864533364772797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,3584,0.011885866522789001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,2560,0.01013973355293274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,65536,0.11792106628417968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,256,0.004990933338801066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,1024,0.0061141331990559895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,768,0.005668266614278158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,512,0.005412266651789347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,128,0.004582400123278299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,16384,0.031351466973622635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,8192,0.01734293301900228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,10240,0.021195733547210695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,7168,0.01649386684099833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,12288,0.02438933253288269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,4096,0.011450666189193725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,6144,0.014509866635004679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,3072,0.010133333007494609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,5120,0.013330133756001792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,2048,0.008038400113582611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,1536,0.006589866677920024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,3584,0.011134933431943257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,768,0.005672533313433329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,2560,0.009593600034713745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,512,0.00517546683549881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,65536,0.1043776035308838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,1024,0.005995733539263407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,8192,0.017492266496022542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,256,0.0047775998711586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,12288,0.02232853372891744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,128,0.004712533454100291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,16384,0.030060799916585286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,5120,0.012421333789825439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,6144,0.013692800203959146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,10240,0.02073600093523661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,7168,0.016080000003178916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,3072,0.009581866860389709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,2560,0.00904746651649475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,4096,0.011527466773986816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,3584,0.010921600461006164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,2048,0.00780266672372818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,65536,0.10200426578521729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,1536,0.0069930667678515124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,1024,0.0057781333724657696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,768,0.005313066641489664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,512,0.0052714665730794275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,256,0.004929066697756449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,128,0.004712533454100291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,12288,0.02090880076090495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,8192,0.016637866695721946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,10240,0.018311466773351034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,16384,0.027628799279530842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,7168,0.01539306640625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,3584,0.010237866640090942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,6144,0.013967999815940857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,4096,0.010955733060836793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,5120,0.011865599950154623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,65536,0.09371946652730306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,3072,0.00944106678167979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,2048,0.0070698668559392285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,2560,0.00855573316415151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,1536,0.006771199901898702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,768,0.005415466427803039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,1024,0.005825066566467285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,128,0.004572799801826477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,512,0.005167999863624572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,256,0.004852266609668731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,16384,0.02758293350537618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,7168,0.01509119967619578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,12288,0.020143999656041463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,10240,0.019050665696461997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,8192,0.015089066823323569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,5120,0.012310399611790975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,4096,0.010890666643778484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,6144,0.013642666737238565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,2048,0.007303466896216075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,1024,0.0058677335580190025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,65536,0.09247786998748779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,3584,0.009973333279291789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,3072,0.009202133615811665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,1536,0.006626133124033611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,2560,0.008097066481908163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,256,0.004969599843025208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,128,0.004655999938646952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,768,0.005605333546797434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,512,0.005202133456865946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,16384,0.02540480097134908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,10240,0.018172800540924072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,7168,0.014362667004267374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,12288,0.019637332359949747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,5120,0.011742933591206869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,4096,0.010309333602587383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,8192,0.014837333559989929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,65536,0.08590826988220215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,2560,0.007923200229803721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,6144,0.012961066762606301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,1536,0.006387199958165486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,1024,0.005704533557097117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,3584,0.009894399841626485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,512,0.0051925331354141235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,3072,0.008303999900817871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,2048,0.007051733136177063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,128,0.004633600016434988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,12288,0.019512534141540527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,768,0.005414400001366933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,256,0.004785066843032837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,10240,0.01714986761411031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,8192,0.014779733618100485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,16384,0.02424853245417277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,65536,0.0811626672744751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,7168,0.0136245330174764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,6144,0.012432000041007996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,5120,0.011108266313870747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,3584,0.008986666798591614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,4096,0.009378133217493693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,1536,0.0064629331231117245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,3072,0.008027733365694682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,2560,0.007665066421031952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,2048,0.006796800096829732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,1024,0.005662933488686879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,512,0.0049781332413355505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,768,0.005335466563701629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,256,0.00470719983180364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,128,0.00456639975309372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,65536,0.08091413180033366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,16384,0.02411839962005615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,10240,0.017042134205500284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,5120,0.010440533359845478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,12288,0.019357866048812865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,6144,0.012004266182581585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,7168,0.013132799665133157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,8192,0.014607999722162882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,3072,0.007993599772453308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,2560,0.007614933451016744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,4096,0.009197866916656494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,3584,0.008986666798591614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,1024,0.005660800139109293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,512,0.005020800232887268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,2048,0.006793599824110668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,1536,0.006545066833496094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,768,0.005322666466236114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,16384,0.024002132813135783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,256,0.004675200084845225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,65536,0.0807925303777059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,128,0.004571733375390371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,6144,0.011506133278210958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,12288,0.019250132640202842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,10240,0.016723199685414632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,8192,0.014297599593798319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,7168,0.013075199723243714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,3072,0.008041599889596303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,5120,0.010386133193969726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,2560,0.007707733412583668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,4096,0.009187199672063192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,2048,0.006788266698519389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,3584,0.00901759962240855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,1024,0.0056202664971351625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,256,0.004701866706212362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,1536,0.006458666423956554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,512,0.004952533543109894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,768,0.005267199873924255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,10240,0.01653866668542226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,12288,0.019019732872645058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,128,0.004604800045490265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,16384,0.023689599831899007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,6144,0.011507200201352437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,5120,0.010366933544476827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,65536,0.08062613010406494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,7168,0.012702932953834534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,3072,0.00795306662718455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,8192,0.013838932911554972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,4096,0.009181867043177288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,1024,0.0056415999929110205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,3584,0.008929066856702169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,2048,0.006775466601053874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,2560,0.007642666498819987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,256,0.004683733483155568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,768,0.0052042668064435325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,1536,0.006515199939409892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,128,0.004564266900221507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,512,0.004970666766166687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,12288,0.018447999159495035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,6144,0.011481600006421407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,65536,0.08028480211893717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,10240,0.01616426706314087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,16384,0.023219199975331624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,8192,0.013795199990272521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,7168,0.012660266955693564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,5120,0.01032960017522176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,1024,0.0055978665749231975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,4096,0.009154133001963298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,3584,0.008942932883898417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,2048,0.006774400174617767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,3072,0.007985066870848339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,1536,0.006404266754786174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,2560,0.007566933333873749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,256,0.004647466540336609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,16384,0.023139200607935586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,65536,0.07969173590342203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,768,0.005176533261934916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,8192,0.013782399892807006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,512,0.004957866668701172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,6144,0.01145919958750407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,128,0.0044821331898371375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,12288,0.018518400192260743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,5120,0.01025493343671163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,10240,0.016200533509254454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,4096,0.009155199925104777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,7168,0.012662399808565775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,3072,0.00798399994770686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,2048,0.006702933212121327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,2560,0.007585066556930542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,3584,0.008778666456540424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,1536,0.006389333307743073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,1024,0.005657599866390228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,768,0.005172266562779745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,256,0.004632533093293508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,512,0.004910933474699656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,128,0.004462933540344239
