tt-metal: [Bug Report] Matmul crashing

Describe the bug Matmul with this parameters results in crash due to limitation of L1 cache. This should never happen.

ttnn::linear(
        tensor->get_value(),   // assume it's just ttnn::Tensor
        weight->get_value(),  // assume it's just ttnn::Tensor
        bias->get_value(),      // assume it's just ttnn::Tensor
        /* transpose_a */ false,
        /* tranpose_b */ true,
        /* memory_config */ std::nullopt,
        /* dtype */ std::nullopt,
        /* program_config */ std::nullopt,
        /* activation */ std::nullopt,
        /* compute_kernel_config */ core::ComputeKernelConfig::matmul(),
        /* core_grid */ ttnn::CoreGrid{7, 8})
ttnn::WormholeComputeKernelConfig ComputeKernelConfig::matmul() {
    ttnn::WormholeComputeKernelConfig config;
    config.fp32_dest_acc_en = false;
    config.math_approx_mode = false;
    config.math_fidelity = MathFidelity::HiFi2;
    config.packer_l1_acc = true;
    return config;
}

BOUND MATMUL: A shape SimpleShape([64, 1, 1024, 768]) B shape SimpleShape([1, 1, 2304, 768]) transpose A false transpose B true

                 Always | FATAL    | Statically allocated circular buffers on core range [(x=0,y=0) - (x=6,y=7)] grow to 6148640 B which is beyond max L1 size of 1499136 B
libc++abi: terminating due to uncaught exception of type std::runtime_error: TT_THROW @ /home/ubuntu/tt-metal/tt_metal/impl/program/program.cpp:819: tt::exception
info:
Statically allocated circular buffers on core range [(x=0,y=0) - (x=6,y=7)] grow to 6148640 B which is beyond max L1 size of 1499136 B
backtrace:
 --- /home/ubuntu/tt-metal/build/lib/libtt_metal.so(+0x12e1bb) [0x7f06ffa301bb]
 --- tt::tt_metal::detail::Program_::validate_circular_buffer_region(tt::tt_metal::v0::Device const*)
 --- tt::tt_metal::v0::CommandQueue::run_command_impl(tt::tt_metal::CommandInterface const&)
 --- tt::tt_metal::v0::EnqueueProgram(tt::tt_metal::v0::CommandQueue&, tt::tt_metal::v0::Program&, bool)
 --- tt::tt_metal::operation::OldInfraDeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>::tensor_return_value_t ttnn::device_operation::detail::launch_on_single_device<tt::tt_metal::operation::OldInfraDeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>>(unsigned char, tt::tt_metal::operation::OldInfraDeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>::operation_attributes_t const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>::tensor_args_t const&)
 --- /home/ubuntu/tt-metal/build/lib/_ttnn.so(+0x1d7c05a) [0x7f06feb8b05a]
 --- tt::tt_metal::operation::OldInfraDeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>::tensor_return_value_t ttnn::device_operation::detail::invoke<tt::tt_metal::operation::OldInfraDeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>>(unsigned char, tt::tt_metal::operation::OldInfraDeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>::operation_attributes_t const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>::tensor_args_t const&)
 --- /home/ubuntu/tt-metal/build/lib/_ttnn.so(+0x1d7b3cb) [0x7f06feb8a3cb]
 --- /home/ubuntu/tt-metal/build/lib/_ttnn.so(+0x1d7820b) [0x7f06feb8720b]
 --- std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> tt::tt_metal::operation::run<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>(tt::tt_metal::operation::DeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>&&, std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor const>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor const>>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor>>> const&, unsigned char)
 --- /home/ubuntu/tt-metal/build/lib/_ttnn.so(+0x13a9e41) [0x7f06fe1b8e41]
 --- /home/ubuntu/tt-metal/build/lib/_ttnn.so(+0x13aab65) [0x7f06fe1b9b65]
 --- ttnn::operations::matmul::matmul(tt::tt_metal::Tensor const&, tt::tt_metal::Tensor const&, std::__1::optional<tt::tt_metal::Tensor const> const&, ttnn::operations::matmul::Matmul const&, unsigned char)
 --- ttnn::operations::matmul::bound_matmul(tt::tt_metal::Tensor const&, tt::tt_metal::Tensor const&, std::__1::optional<tt::tt_metal::Tensor const> const&, ttnn::operations::matmul::Matmul const&, unsigned char const&)
 --- /home/ubuntu/tt-metal/build/lib/_ttnn.so(_ZN4ttnn10operations6matmul15LinearOperation6invokeERKN2tt8tt_metal6TensorES7_RKNSt3__18optionalIS6_EEbbRKNS9_IKNS4_12MemoryConfigEEENS9_IKNS4_8DataTypeEEERKNS9_IKNS8_7variantIJNS1_28MatmulMultiCoreProgramConfigENS1_45MatmulMultiCoreNonOptimizedReuseProgramConfigENS1_33MatmulMultiCoreReuseProgramConfigENS1_42MatmulMultiCoreReuseMultiCastProgramConfigENS1_44MatmulMultiCoreReuseMultiCast1DProgramConfigENS1_53MatmulMultiCoreReuseMultiCastDRAMShardedProgramConfigEEEEEERKNS9_IKNS8_12basic_stringIcNS8_11char_traitsIcEENS8_9allocatorIcEEEEEENS9_IKNSL_IJNS_28GrayskullComputeKernelConfigENS_27WormholeComputeKernelConfigEEEEEENS9_IKNS_5types8CoreGridEEERKNS9_IKNS4_4TileEEE+0x2cd) [0x7f06fe188f2d]
 --- /home/ubuntu/tt-metal/tt-train/build/sources/examples/nano_gpt/nano_gpt(+0x207326) [0x555e9e605326]
 --- /home/ubuntu/tt-metal/tt-train/build/sources/examples/nano_gpt/nano_gpt(+0x202111) [0x555e9e600111]
 --- /home/ubuntu/tt-metal/tt-train/build/sources/examples/nano_gpt/nano_gpt(+0x200ec5) [0x555e9e5feec5]
 --- /home/ubuntu/tt-metal/tt-train/build/sources/examples/nano_gpt/nano_gpt(+0x194c3a) [0x555e9e592c3a]
 --- /home/ubuntu/tt-metal/tt-train/build/sources/examples/nano_gpt/nano_gpt(+0x1951c8) [0x555e9e5931c8]
 --- /home/ubuntu/tt-metal/tt-train/build/sources/examples/nano_gpt/nano_gpt(+0x193c11) [0x555e9e591c11]
 --- /home/ubuntu/tt-metal/tt-train/build/sources/examples/nano_gpt/nano_gpt(+0x18db1e) [0x555e9e58bb1e]
 --- /home/ubuntu/tt-metal/tt-train/build/sources/examples/nano_gpt/nano_gpt(+0xf7f3b) [0x555e9e4f5f3b]
 --- /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf3) [0x7f06fc42c083]
 --- /home/ubuntu/tt-metal/tt-train/build/sources/examples/nano_gpt/nano_gpt(+0xf413e) [0x555e9e4f213e]

Aborted (core dumped)

To Reproduce There are two ways to reproduce issue.

  1. Run tt-train with config training_shakespear_gpt2s.yaml (increase batch size to 64).
  2. Implement minimalistic python program which multiplies specified matrices (with given cores and compute config parameters)

Expected behavior If one of the algorithms not able to perform due to hardware limitations it should fallback to one that works.

Please complete the following environment information:

  • OS: Ubuntu 20.04
  • Version of software: 84401ed5f3c516798d978c3fcc64d843b4fbec82

About this issue

  • Original URL
  • State: closed
  • Created 2 years ago
  • Comments: 37 (34 by maintainers)

Commits related to this issue

Most upvoted comments

Found the issue. The amount to reserve by was not adjusted to be block sized, but the CB size was adjusted. Therefore there was not enough space to reserve the requested number of tiles.

The following fixes the problem:

diff --git a/ttnn/cpp/ttnn/operations/matmul/device/matmul_op_multi_core_reuse_mcast_1d_program_factory.cpp b/ttnn/c
pp/ttnn/operations/matmul/device/matmul_op_multi_core_reuse_mcast_1d_program_factory.cpp
index 4e29985533..a701cbf663 100644
--- a/ttnn/cpp/ttnn/operations/matmul/device/matmul_op_multi_core_reuse_mcast_1d_program_factory.cpp
+++ b/ttnn/cpp/ttnn/operations/matmul/device/matmul_op_multi_core_reuse_mcast_1d_program_factory.cpp
@@ -1229,7 +1229,7 @@ operation::ProgramWithCallbacks create_program_mcast_in1(
     };
 
     if (bias_buffer != nullptr) {
-        in1_receiver_writer_compile_time_args.push_back((std::uint32_t)per_core_N);
+        in1_receiver_writer_compile_time_args.push_back((std::uint32_t)in1_block_w);
     }

Next steps:

  • run the tests again and see if there are any other issues.

Update:

  • still haven’t found source of hangs
  • after that need to update tests/models to pass pipelines

Hey @bbradelTT! Thanks for looking into the trace. I’ve also parsed the trace. Please take a look at ttnn::matmul and ttnn::linear (shape for second input should be transposed). It contains all unique operation calls.

https://docs.google.com/spreadsheets/d/1bkeeVeW_2YCMmO9DokcgyY1XNDckgoyYIbKxBxrXVCw/edit?gid=1071015504#gid=1071015504

Image

@bbradelTT I don’t think setting program config is an easy solution/task. Here is an example of all configs that groq models (found in models) use (just for matmul):

        self.model_config["ROT_MAT_MM_PROGCFG"] = ttnn.MatmulMultiCoreReuseMultiCast1DProgramConfig(
            compute_with_storage_grid_size=(8, 4),
            in0_block_w=4,
            out_subblock_h=1,
            out_subblock_w=4,
            per_core_M=1,
            per_core_N=4,
            fuse_batch=True,
            fused_activation=None,
            mcast_in0=False,
        )

        self.model_config["GATE_MM_OUTPUT_PROGCFG"] = ttnn.MatmulMultiCoreReuseMultiCast1DProgramConfig(
            compute_with_storage_grid_size=(8, 1),
            in0_block_w=24,
            out_subblock_h=1,
            out_subblock_w=1,
            per_core_M=1,
            per_core_N=1,
            fuse_batch=True,
            fused_activation=None,
            mcast_in0=False,
        )

        self.model_config["QKV_MM_OUTPUT_PROGCFG"] = ttnn.MatmulMultiCoreReuseMultiCast1DProgramConfig(
            compute_with_storage_grid_size=(8, 4),
            in0_block_w=6,
            out_subblock_h=1,
            out_subblock_w=1,
            per_core_M=1,
            per_core_N=1,
            fuse_batch=True,
            fused_activation=None,
            mcast_in0=True,
        )

        self.model_config["SCORES_BATCHED_MM_PROGCFG"] = cached_lambda(
            lambda p: ttnn.MatmulMultiCoreReuseProgramConfig(
                compute_with_storage_grid_size=(8, 4),
                in0_block_w=4,
                out_subblock_h=1,
                out_subblock_w=1,
                per_core_M=1,
                per_core_N=p,
            )
        )

        self.model_config["VALUES_BATCHED_MM_PROGCFG"] = cached_lambda(
            lambda p: ttnn.MatmulMultiCoreReuseProgramConfig(
                compute_with_storage_grid_size=(8, 4),
                in0_block_w=p,
                out_subblock_h=1,
                out_subblock_w=4,
                per_core_M=1,
                per_core_N=4,
            )
        )

        self.model_config["LM_HEAD_OUTPUT_PROGCFG"] = ttnn.MatmulMultiCoreReuseMultiCast1DProgramConfig(
            compute_with_storage_grid_size=(8, 8),
            in0_block_w=1,
            out_subblock_h=1,
            out_subblock_w=3,
            per_core_M=1,
            per_core_N=3,
            fuse_batch=True,
            fused_activation=None,
            mcast_in0=True,
        )

        self.model_config["FF1_OUTPUT_PROGCFG"] = ttnn.MatmulMultiCoreReuseMultiCast1DProgramConfig(
            compute_with_storage_grid_size=(8, 8),
            in0_block_w=2,  # K = 6144 / TILE_WIDTH=32 / Grid_Size is based on compute_with_storage_grid_size
            out_subblock_h=1,  # Must be divisible by per_core_M
            out_subblock_w=4,  # Must be divisible by per_core_N, out_subblock_w * out_subblock_h <= 4
            per_core_M=1,  # M / TILE_HEIGHT = 32 / 32
            per_core_N=16,  # N / TILE_WIDTH / Grid_Size is based on compute_with_storage_grid_size, N = 32768
            fuse_batch=True,
            fused_activation=(ttnn.UnaryOpType.GELU, True),  # FIXME: GET THIS DOCUMENTED
            mcast_in0=True,
        )

        self.model_config["FF3_OUTPUT_PROGCFG"] = ttnn.MatmulMultiCoreReuseMultiCast1DProgramConfig(
            compute_with_storage_grid_size=(8, 8),
            in0_block_w=2,  # K = 6144 / TILE_WIDTH=32 / Grid_Size is based on compute_with_storage_grid_size
            out_subblock_h=1,  # Must be divisible by per_core_M
            out_subblock_w=4,  # Must be divisible by per_core_N, out_subblock_w * out_subblock_h <= 4
            per_core_M=1,  # M / TILE_HEIGHT = 32 / 32
            per_core_N=16,  # N / TILE_WIDTH / Grid_Size is based on compute_with_storage_grid_size, N = 32768
            fuse_batch=True,
            fused_activation=None,
            mcast_in0=True,
        )

        self.model_config["FF2_OUTPUT_PROGCFG"] = ttnn.MatmulMultiCoreReuseMultiCast1DProgramConfig(
            compute_with_storage_grid_size=(8, 8),
            in0_block_w=16,  # K = 32768 / TILE_WIDTH=32 / Grid_Size is based on compute_with_storage_grid_size
            out_subblock_h=1,  # Must be divisible by per_core_M
            # Issue #8959: Increasing subblock to 2 results in hangs -> Potentially related to di/dt hangs.
            out_subblock_w=3,  # Must be divisible by per_core_N, out_subblock_w * out_subblock_h <= 4
            per_core_M=1,  # M / TILE_HEIGHT = 32 / 32
            per_core_N=3,  # N / TILE_WIDTH / Grid_Size is based on compute_with_storage_grid_size, N = 6144
            fuse_batch=True,
            fused_activation=None,
            mcast_in0=True,
        )

        self.model_config["OUTPUT_MM_PROGCFG"] = ttnn.MatmulMultiCoreReuseMultiCast1DProgramConfig(
            compute_with_storage_grid_size=(7, 6),  # TODO Hanging with full coreGrid (8,8)
            in0_block_w=2,
            out_subblock_h=1,
            out_subblock_w=4,
            per_core_M=1,
            per_core_N=32,
            fuse_batch=True,
            fused_activation=None,
            mcast_in0=True,
        )

I was able to re-create the issue via a pytest:

import pytest
import torch
import ttnn
from tests.ttnn.utils_for_testing import assert_with_pcc
from models.utility_functions import torch_random


def test_mm2(device):
    x = torch.ones((64, 1, 1024, 768), dtype=torch.float32)
    #y = torch.ones((1, 1, 768, 2304), dtype=torch.float32)
    y = torch.ones((1, 1, 2304, 768), dtype=torch.float32)
    x_tt = ttnn.from_torch(x, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=device)
    y_tt = ttnn.from_torch(y, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=device)
    #z_tt = ttnn.matmul(x_tt, y_tt, core_grid=ttnn.CoreGrid(y=7, x=8))
    z_tt = ttnn.matmul(x_tt, y_tt, transpose_b=True, core_grid=ttnn.CoreGrid(y=7, x=8))
    z = ttnn.to_torch(z_tt)
    z_t = torch.matmul(x, y)
    print(f'z_t {z_t}\nz_tt {z_tt}\nz {z}')
    assert_with_pcc(z_t, z)

I’ll work on checking expected CB sizes vs L1 and adjust parameters accordingly in more scenarios than we currently do.

  • Input and output tensors are in DRAM
  • Does Matmul under the hood reshapes to rank-2, ie in0.reshape(-1, 768) to (65536, 768) ?
  • Isn’t this shape covered with latest work by @yugaoTT – output tensor is (65536, 2304) and this we could probably height shard across cores and have multiple output blocks , and still use 8x8 cores

fyi @TT-BrianLiu

This is a blocker for TT-train, does P0

I was able to re-create the issue via a pytest:

import pytest
import torch
import ttnn
from tests.ttnn.utils_for_testing import assert_with_pcc
from models.utility_functions import torch_random


def test_mm2(device):
    x = torch.ones((64, 1, 1024, 768), dtype=torch.float32)
    #y = torch.ones((1, 1, 768, 2304), dtype=torch.float32)
    y = torch.ones((1, 1, 2304, 768), dtype=torch.float32)
    x_tt = ttnn.from_torch(x, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=device)
    y_tt = ttnn.from_torch(y, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=device)
    #z_tt = ttnn.matmul(x_tt, y_tt, core_grid=ttnn.CoreGrid(y=7, x=8))
    z_tt = ttnn.matmul(x_tt, y_tt, transpose_b=True, core_grid=ttnn.CoreGrid(y=7, x=8))
    z = ttnn.to_torch(z_tt)
    z_t = torch.matmul(x, y)
    print(f'z_t {z_t}\nz_tt {z_tt}\nz {z}')
    assert_with_pcc(z_t, z)

I’ll work on checking expected CB sizes vs L1 and adjust parameters accordingly in more scenarios than we currently do.

  • Input and output tensors are in DRAM
  • Does Matmul under the hood reshapes to rank-2, ie in0.reshape(-1, 768) to (65536, 768) ?
  • Isn’t this shape covered with latest work by @yugaoTT – output tensor is (65536, 2304) and this we could probably height shard across cores and have multiple output blocks , and still use 8x8 cores.
  • we would have each core do (1024,2304) , and then have multiple blocks, say block size (256,256), then there are 4 & 9 blocks in height and width
  • heuristics for handling large in/out Tensors in DRAM: pick height / width / block parallelization based on the shape (tall, wide, or square-ish), then pick the block size per core and number of blocks in W / H

fyi @TT-BrianLiu

I was able to re-create the issue via a pytest:

import pytest
import torch
import ttnn
from tests.ttnn.utils_for_testing import assert_with_pcc
from models.utility_functions import torch_random


def test_mm2(device):
    x = torch.ones((64, 1, 1024, 768), dtype=torch.float32)
    #y = torch.ones((1, 1, 768, 2304), dtype=torch.float32)
    y = torch.ones((1, 1, 2304, 768), dtype=torch.float32)
    x_tt = ttnn.from_torch(x, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=device)
    y_tt = ttnn.from_torch(y, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=device)
    #z_tt = ttnn.matmul(x_tt, y_tt, core_grid=ttnn.CoreGrid(y=7, x=8))
    z_tt = ttnn.matmul(x_tt, y_tt, transpose_b=True, core_grid=ttnn.CoreGrid(y=7, x=8))
    z = ttnn.to_torch(z_tt)
    z_t = torch.matmul(x, y)
    print(f'z_t {z_t}\nz_tt {z_tt}\nz {z}')
    assert_with_pcc(z_t, z)

I’ll work on checking expected CB sizes vs L1 and adjust parameters accordingly in more scenarios than we currently do.