tt-mlir: ttrt perf fails on const-eval graph
ttrt perf fails on the following const-eval graph:
#dram = #ttnn.buffer_type<dram>
#system_desc = #tt.system_desc<[{role = host, target_triple = "x86_64-pc-linux"}], [{arch = <wormhole_b0>, grid = 8x8, coord_translation_offsets = 18x18, l1_size = 1499136, num_dram_channels = 12, dram_channel_size = 1073741824, noc_l1_address_align_bytes = 16, pcie_address_align_bytes = 32, noc_dram_address_align_bytes = 32, l1_unreserved_base = 97248, erisc_l1_unreserved_base = 69632, dram_unreserved_base = 32, dram_unreserved_end = 1073158336, physical_helper_cores = {dram = [ 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10, 0x11] eth_inactive = [ 16x18, 16x19, 16x20, 16x21, 16x22, 16x23, 16x24, 16x25, 17x19, 17x20, 17x22, 17x23, 17x24]}, supported_data_types = [<f32>, <f16>, <bf16>, <bfp_f8>, <bfp_bf8>, <bfp_f4>, <bfp_bf4>, <bfp_f2>, <bfp_bf2>, <u32>, <u16>, <u8>, <si32>], supported_tile_sizes = [ 4x16, 16x16, 32x16, 4x32, 16x32, 32x32], num_cbs = 32, num_compute_threads = 1, num_datamovement_threads = 2}], [0], [3 : i32], [ 0x0x0x0]>
#system_memory = #ttnn.buffer_type<system_memory>
#ttnn_layout = #ttnn.ttnn_layout<(d0, d1, d2, d3) -> (d0 * 2048 + d1 * 32 + d2, d3), <1x1>, memref<4096x1x!tt.tile<32x32, f32>, #dram>, <interleaved>>
#ttnn_layout1 = #ttnn.ttnn_layout<(d0, d1, d2, d3) -> (d0 * 32 + d1 * 32 + d2, d3), <1x1>, memref<1x2x!tt.tile<32x32, f32>, #dram>, <interleaved>>
#ttnn_layout2 = #ttnn.ttnn_layout<(d0, d1, d2, d3) -> (d0 * 576 + d1 * 576 + d2, d3), <8x8, (d0, d1) -> (0, d0, d1)>, memref<3x1x!tt.tile<32x32, f32>, #dram>, <interleaved>>
#ttnn_layout3 = #ttnn.ttnn_layout<(d0, d1, d2, d3) -> (d0 * 32 + d1 * 32 + d2, d3), <1x1>, memref<64x1x!tt.tile<32x32, f32>, #dram>, <interleaved>>
#ttnn_layout4 = #ttnn.ttnn_layout<(d0, d1, d2, d3) -> (d0 * 2048 + d1 * 32 + d2, d3), <1x1>, memref<4096x1x!tt.tile<32x32, f32>, #system_memory>>
#ttnn_layout5 = #ttnn.ttnn_layout<(d0, d1, d2, d3) -> (d0 * 192 + d1 * 3 + d2, d3), <1x1>, memref<12288x3xf32, #system_memory>>
#ttnn_layout6 = #ttnn.ttnn_layout<(d0, d1, d2, d3) -> (d0 * 3584 + d1 * 64 + d2, d3), <1x1>, memref<112x2x!tt.tile<32x32, f32>, #dram>, <interleaved>>
#ttnn_layout7 = #ttnn.ttnn_layout<(d0, d1, d2, d3) -> (d0 * 3584 + d1 * 64 + d2, d3), <8x8, (d0, d1) -> (0, d0, d1)>, memref<14x1x!tt.tile<32x32, f32>, #dram>, <interleaved>>
#ttnn_layout8 = #ttnn.ttnn_layout<(d0, d1, d2, d3) -> (d0 * 3136 + d1 * 3136 + d2, d3), <8x8, (d0, d1) -> (0, d0, d1)>, memref<13x1x!tt.tile<32x32, f32>, #dram>, <interleaved>>
module {
tt.device_module {
builtin.module attributes {tt.system_desc = #system_desc} {
tt.device @default_device = <workerGrid = #tt.grid<8x8, (d0, d1) -> (0, d0, d1)>, l1Map = (d0, d1, d2)[s0] -> (0, d0, d1, d2 + s0), dramMap = (d0, d1, d2)[s0, s1, s2, s3, s4, s5] -> (0, 0, (((d0 * s1) * (s2 * s3) + d1 * (s2 * s3) + d2) floordiv s4) mod 12, ((d0 * s1) * (s2 * s3) + d1 * (s2 * s3) + d2) floordiv (s4 * 12) + ((d0 * s1) * (s2 * s3) + d1 * (s2 * s3) + d2) mod s4 + s5), meshShape = , chipIds = [0]>
func.func @main_const_eval_0(%arg0: tensor<64x64x3x3xf32, #ttnn_layout>, %arg1: tensor<1x1x1x64xf32, #ttnn_layout1>) -> tensor<1x1x576x64xf32, #ttnn_layout2> attributes {const_eval} {
%0 = "ttnn.get_device"() <{mesh_offset = #ttnn<mesh_offset 0x0>, mesh_shape = #ttnn<mesh_shape 1x1>}> : () -> !ttnn.device
%1 = "ttnn.reshape"(%arg1) <{shape = [64 : i32, 1 : i32, 1 : i32, 1 : i32]}> : (tensor<1x1x1x64xf32, #ttnn_layout1>) -> tensor<64x1x1x1xf32, #ttnn_layout3>
%2 = "ttnn.multiply"(%arg0, %1) : (tensor<64x64x3x3xf32, #ttnn_layout>, tensor<64x1x1x1xf32, #ttnn_layout3>) -> tensor<64x64x3x3xf32, #ttnn_layout>
"ttnn.deallocate"(%1) <{force = false}> : (tensor<64x1x1x1xf32, #ttnn_layout3>) -> ()
%3 = "ttnn.from_device"(%2) : (tensor<64x64x3x3xf32, #ttnn_layout>) -> tensor<64x64x3x3xf32, #ttnn_layout4>
"ttnn.deallocate"(%2) <{force = false}> : (tensor<64x64x3x3xf32, #ttnn_layout>) -> ()
%4 = "ttnn.to_layout"(%3) <{layout = #ttnn.layout<row_major>}> : (tensor<64x64x3x3xf32, #ttnn_layout4>) -> tensor<64x64x3x3xf32, #ttnn_layout5>
"ttnn.deallocate"(%3) <{force = false}> : (tensor<64x64x3x3xf32, #ttnn_layout4>) -> ()
%5 = "ttnn.prepare_conv2d_weights"(%4, %0) <{batch_size = 1 : i32, conv2d_config = #ttnn.conv2d_config<dtype = f32, weights_dtype = f32, activation = "relu", deallocate_activation = false, reallocate_halo_output = true, act_block_h_override = 0, act_block_w_div = 1, reshard_if_not_optimal = false, override_sharding_config = false, transpose_shards = true, output_layout = tile, preprocess_weights_on_device = false, always_preprocess_weights = false, enable_act_double_buffer = false, enable_weights_double_buffer = false, enable_split_reader = false, enable_subblock_padding = false>, dilation = array<i32: 1, 1>, groups = 1 : i32, has_bias = true, in_channels = 64 : i32, input_height = 56 : i32, input_memory_config = #ttnn.memory_config<#dram, <interleaved>>, input_tensor_layout = #ttnn.layout<tile>, input_width = 56 : i32, kernel_size = array<i32: 3, 3>, out_channels = 64 : i32, padding = array<i32: 1, 1>, stride = array<i32: 1, 1>, weights_format = "OIHW"}> : (tensor<64x64x3x3xf32, #ttnn_layout5>, !ttnn.device) -> tensor<1x1x576x64xf32, #ttnn_layout2>
"ttnn.deallocate"(%4) <{force = false}> : (tensor<64x64x3x3xf32, #ttnn_layout5>) -> ()
return %5 : tensor<1x1x576x64xf32, #ttnn_layout2>
}
func.func @main(%arg0: tensor<64x64x3x3xf32, #ttnn_layout> {tt.argument_type = #tt.argument_type<constant>}, %arg1: tensor<1x1x1x64xf32, #ttnn_layout1> {tt.argument_type = #tt.argument_type<constant>}, %arg2: tensor<1x1x1x64xf32, #ttnn_layout1>, %arg3: tensor<1x56x56x64xf32, #ttnn_layout6>) -> tensor<1x56x56x64xf32, #ttnn_layout7> {
%0 = tt.load_cached(@main_const_eval_0, [%arg0, %arg1]) : (tensor<64x64x3x3xf32, #ttnn_layout>, tensor<1x1x1x64xf32, #ttnn_layout1>) -> tensor<1x1x576x64xf32, #ttnn_layout2>
"ttnn.deallocate"(%arg1) <{force = false}> : (tensor<1x1x1x64xf32, #ttnn_layout1>) -> ()
"ttnn.deallocate"(%arg0) <{force = false}> : (tensor<64x64x3x3xf32, #ttnn_layout>) -> ()
%1 = "ttnn.get_device"() <{mesh_offset = #ttnn<mesh_offset 0x0>, mesh_shape = #ttnn<mesh_shape 1x1>}> : () -> !ttnn.device
%2 = "ttnn.reshape"(%arg3) <{shape = [1 : i32, 1 : i32, 3136 : i32, 64 : i32]}> : (tensor<1x56x56x64xf32, #ttnn_layout6>) -> tensor<1x1x3136x64xf32, #ttnn_layout8>
"ttnn.deallocate"(%arg3) <{force = false}> : (tensor<1x56x56x64xf32, #ttnn_layout6>) -> ()
%3 = "ttnn.conv2d"(%2, %0, %arg2, %1) <{batch_size = 1 : i32, conv2d_config = #ttnn.conv2d_config<dtype = f32, weights_dtype = f32, activation = "relu", deallocate_activation = false, reallocate_halo_output = true, act_block_h_override = 0, act_block_w_div = 1, reshard_if_not_optimal = false, override_sharding_config = false, transpose_shards = true, output_layout = tile, preprocess_weights_on_device = false, always_preprocess_weights = false, enable_act_double_buffer = false, enable_weights_double_buffer = false, enable_split_reader = false, enable_subblock_padding = false>, dilation = array<i32: 1, 1>, groups = 1 : i32, in_channels = 64 : i32, input_height = 56 : i32, input_width = 56 : i32, kernel_size = array<i32: 3, 3>, out_channels = 64 : i32, padding = array<i32: 1, 1>, stride = array<i32: 1, 1>}> : (tensor<1x1x3136x64xf32, #ttnn_layout8>, tensor<1x1x576x64xf32, #ttnn_layout2>, tensor<1x1x1x64xf32, #ttnn_layout1>, !ttnn.device) -> tensor<1x1x3136x64xf32, #ttnn_layout8>
"ttnn.deallocate"(%2) <{force = false}> : (tensor<1x1x3136x64xf32, #ttnn_layout8>) -> ()
"ttnn.deallocate"(%0) <{force = false}> : (tensor<1x1x576x64xf32, #ttnn_layout2>) -> ()
"ttnn.deallocate"(%arg2) <{force = false}> : (tensor<1x1x1x64xf32, #ttnn_layout1>) -> ()
%4 = "ttnn.reshape"(%3) <{shape = [1 : i32, 56 : i32, 56 : i32, 64 : i32]}> : (tensor<1x1x3136x64xf32, #ttnn_layout8>) -> tensor<1x56x56x64xf32, #ttnn_layout7>
"ttnn.deallocate"(%3) <{force = false}> : (tensor<1x1x3136x64xf32, #ttnn_layout8>) -> ()
return %4 : tensor<1x56x56x64xf32, #ttnn_layout7>
}
}
}
}
Error Log:
2025-05-19 10:00:22,815 - INFO - host side ops time report generated at tracy_ops_times.csv
2025-05-19 10:00:22,820 - INFO - host side ops data report generated at tracy_ops_data.csv
2025-05-19 10:00:22,821 - INFO - saving binary=/localdev/odjuricic/repos/tt-mlir/build/test/ttmlir/Silicon/TTNN/n150/fusing/Output/resnet_pattern_fusing.mlir.tmp.ttnn to binary_folder=/localdev/odjuricic/repos/tt-mlir/ttrt-artifacts/resnet_pattern_fusing.mlir.tmp.ttnn
2025-05-19 10:00:22,821 - INFO - saving program=0 for binary=/localdev/odjuricic/repos/tt-mlir/build/test/ttmlir/Silicon/TTNN/n150/fusing/Output/resnet_pattern_fusing.mlir.tmp.ttnn to program_folder=/localdev/odjuricic/repos/tt-mlir/ttrt-artifacts/resnet_pattern_fusing.mlir.tmp.ttnn/run/program_0
2025-05-19 10:00:22,821 - INFO - saving program=1 for binary=/localdev/odjuricic/repos/tt-mlir/build/test/ttmlir/Silicon/TTNN/n150/fusing/Output/resnet_pattern_fusing.mlir.tmp.ttnn to program_folder=/localdev/odjuricic/repos/tt-mlir/ttrt-artifacts/resnet_pattern_fusing.mlir.tmp.ttnn/run/program_1
2025-05-19 10:00:22,822 - INFO - saving system_desc=system_desc.ttsys to system_desc_folder=/localdev/odjuricic/repos/tt-mlir/ttrt-artifacts/resnet_pattern_fusing.mlir.tmp.ttnn
2025-05-19 10:00:22.824 | INFO | tt_metal.tools.profiler.process_ops_logs:import_tracy_op_logs:108 - Importing ops logs
2025-05-19 10:00:22.848 | INFO | tt_metal.tools.profiler.process_ops_logs:get_device_op_data:252 - Getting device ops
2025-05-19 10:00:22.848 | INFO | tt_metal.tools.profiler.process_ops_logs:append_device_data:289 - Appending device data
2025-05-19 10:00:22.869 | INFO | tt_metal.tools.profiler.process_device_log:core_to_device_timeseries:418 - Importing Data For Device Number : 0
2025-05-19 10:00:23,002 - ERROR - ERROR: test=/localdev/odjuricic/repos/tt-mlir/build/test/ttmlir/Silicon/TTNN/n150/fusing/Output/resnet_pattern_fusing.mlir.tmp.ttnn experienced an error with exception=Device data mismatch: Expected 17 but received 3 ops on device 0. Device is showing op ID 15360 when host is showing op ID 1024
Traceback (most recent call last):
File "/opt/ttmlir-toolchain/venv/lib/python3.10/site-packages/ttrt/common/perf.py", line 513, in _execute
process_ops(None, None, False)
File "/localdev/odjuricic/repos/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/tools/profiler/process_ops_logs.py", line 916, in process_ops
deviceOps, traceOps = append_device_data(ops, traceReplays, logFolder, analyze_noc_traces)
File "/localdev/odjuricic/repos/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/tools/profiler/process_ops_logs.py", line 376, in append_device_data
assert False, (
AssertionError: Device data mismatch: Expected 17 but received 3 ops on device 0. Device is showing op ID 15360 when host is showing op ID 1024
2025-05-19 10:00:23,004 - WARNING - no binaries found to run - returning early
2025-05-19 10:00:23,004 - ERROR - ERROR: test case=/localdev/odjuricic/repos/tt-mlir/build/test/ttmlir/Silicon/TTNN/n150/fusing/Output/resnet_pattern_fusing.mlir.tmp.ttnn
2025-05-19 10:00:23,004 - INFO - results saved to=perf_results.json
About this issue
- Original URL
- State: closed
- Created 2 years ago
- Comments: 16 (11 by maintainers)
@vwellsTT Not super urgent, but is blocking us from measuring device perf.
In general, the wanted behaviour for ttrt perf with const-eval is that we only want result for functions that are not const evaled. For ops that are const-evaled, not perf data is needed. @tapspatel Not sure what the current behaviour is but maybe this helps with the bug as well.
Unfortunately, this is not an easy fix, it will require metal changes to their perf tooling or hacky changes in ttrt. ttrt perf should work now with cached functions and a main function (since cached functions will just be skipped) so we can look into natively supporting multiple functions but you can just separate it into two flatbuffer to get the perf data as a workaround for now. If this becomes cumbersome, we will prioritize it.
ttrt perf doesn’t work with multiple programs currently, so the behavior I believe before Vincent’s fix was it will run the cached program and then run the main program.
I’ll track that here: https://github.com/tenstorrent/tt-mlir/issues/3585. thanks everyone.
We can close this since #3578 is merged