From 51d1e005e86f28450d89c8f272774dc2b74eabf7 Mon Sep 17 00:00:00 2001 From: Samuel Pitoiset Date: Fri, 31 May 2024 09:53:51 +0200 Subject: [PATCH] radv: use the common SQTT implementation I have verified the generated command stream using PM4 is similar to the previous one on POLARIS10, VEGA10, NAVI21 and NAVI31. Signed-off-by: Samuel Pitoiset Part-of: --- src/amd/vulkan/radv_sqtt.c | 329 ++++--------------------------------- 1 file changed, 28 insertions(+), 301 deletions(-) diff --git a/src/amd/vulkan/radv_sqtt.c b/src/amd/vulkan/radv_sqtt.c index ee9f159229c..e35713edc51 100644 --- a/src/amd/vulkan/radv_sqtt.c +++ b/src/amd/vulkan/radv_sqtt.c @@ -15,6 +15,8 @@ #include "radv_sqtt.h" #include "sid.h" +#include "ac_pm4.h" + #include "vk_command_pool.h" #include "vk_common_entrypoints.h" @@ -65,328 +67,53 @@ static void radv_emit_sqtt_start(const struct radv_device *device, struct radeon_cmdbuf *cs, enum radv_queue_family qf) { const struct radv_physical_device *pdev = radv_device_physical(device); - const enum amd_gfx_level gfx_level = pdev->info.gfx_level; - uint32_t shifted_size = device->sqtt.buffer_size >> SQTT_BUFFER_ALIGN_SHIFT; - const struct radeon_info *gpu_info = &pdev->info; - const unsigned shader_mask = ac_sqtt_get_shader_mask(gpu_info); - unsigned max_se = gpu_info->max_se; + const bool is_compute_queue = qf == RADV_QUEUE_COMPUTE; + struct ac_pm4_state *pm4; - radeon_check_space(device->ws, cs, 6 + max_se * 33); + pm4 = ac_pm4_create_sized(&pdev->info, false, 512, is_compute_queue); + if (!pm4) + return; - for (unsigned se = 0; se < max_se; se++) { - uint64_t data_va = ac_sqtt_get_data_va(gpu_info, &device->sqtt, se); - uint64_t shifted_va = data_va >> SQTT_BUFFER_ALIGN_SHIFT; - int active_cu = ac_sqtt_get_active_cu(&pdev->info, se); + ac_sqtt_emit_start(&pdev->info, pm4, &device->sqtt, is_compute_queue); + ac_pm4_finalize(pm4); - if (ac_sqtt_se_is_disabled(gpu_info, se)) - continue; + radeon_check_space(device->ws, cs, pm4->ndw); + radeon_emit_array(cs, pm4->pm4, pm4->ndw); - /* Target SEx and SH0. */ - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, - S_030800_SE_INDEX(se) | S_030800_SH_INDEX(0) | S_030800_INSTANCE_BROADCAST_WRITES(1)); - - if (pdev->info.gfx_level >= GFX11) { - /* Order seems important for the following 2 registers. */ - radeon_set_uconfig_perfctr_reg(gfx_level, qf, cs, R_0367A4_SQ_THREAD_TRACE_BUF0_SIZE, - S_0367A4_SIZE(shifted_size) | S_0367A4_BASE_HI(shifted_va >> 32)); - - radeon_set_uconfig_perfctr_reg(gfx_level, qf, cs, R_0367A0_SQ_THREAD_TRACE_BUF0_BASE, shifted_va); - - uint32_t sqtt_token_mask = S_0367B8_REG_INCLUDE(V_0367B8_REG_INCLUDE_SQDEC | V_0367B8_REG_INCLUDE_SHDEC | - V_0367B8_REG_INCLUDE_GFXUDEC | V_0367B8_REG_INCLUDE_COMP | - V_0367B8_REG_INCLUDE_CONTEXT | V_0367B8_REG_INCLUDE_CONFIG); - - /* Performance counters with SQTT are considered deprecated. */ - uint32_t token_exclude = V_0367B8_TOKEN_EXCLUDE_PERF; - - if (!device->sqtt.instruction_timing_enabled) { - /* Reduce SQTT traffic when instruction timing isn't enabled. */ - token_exclude |= V_0367B8_TOKEN_EXCLUDE_VMEMEXEC | V_0367B8_TOKEN_EXCLUDE_ALUEXEC | - V_0367B8_TOKEN_EXCLUDE_VALUINST | V_0367B8_TOKEN_EXCLUDE_IMMEDIATE | - V_0367B8_TOKEN_EXCLUDE_INST; - } - sqtt_token_mask |= S_0367B8_TOKEN_EXCLUDE_GFX11(token_exclude) | S_0367B8_BOP_EVENTS_TOKEN_INCLUDE_GFX11(1); - - radeon_set_uconfig_perfctr_reg_seq(gfx_level, qf, cs, R_0367B4_SQ_THREAD_TRACE_MASK, 2); - radeon_emit(cs, S_0367B4_WTYPE_INCLUDE(shader_mask) | S_0367B4_SA_SEL(0) | S_0367B4_WGP_SEL(active_cu / 2) | - S_0367B4_SIMD_SEL(0)); - radeon_emit(cs, sqtt_token_mask); - - /* Should be emitted last (it enables thread traces). */ - radeon_set_uconfig_perfctr_reg(gfx_level, qf, cs, R_0367B0_SQ_THREAD_TRACE_CTRL, - ac_sqtt_get_ctrl(&pdev->info, true)); - - } else if (pdev->info.gfx_level >= GFX10) { - /* Order seems important for the following 2 registers. */ - radeon_set_privileged_config_reg(cs, R_008D04_SQ_THREAD_TRACE_BUF0_SIZE, - S_008D04_SIZE(shifted_size) | S_008D04_BASE_HI(shifted_va >> 32)); - - radeon_set_privileged_config_reg(cs, R_008D00_SQ_THREAD_TRACE_BUF0_BASE, shifted_va); - - radeon_set_privileged_config_reg(cs, R_008D14_SQ_THREAD_TRACE_MASK, - S_008D14_WTYPE_INCLUDE(shader_mask) | S_008D14_SA_SEL(0) | - S_008D14_WGP_SEL(active_cu / 2) | S_008D14_SIMD_SEL(0)); - - uint32_t sqtt_token_mask = S_008D18_REG_INCLUDE(V_008D18_REG_INCLUDE_SQDEC | V_008D18_REG_INCLUDE_SHDEC | - V_008D18_REG_INCLUDE_GFXUDEC | V_008D18_REG_INCLUDE_COMP | - V_008D18_REG_INCLUDE_CONTEXT | V_008D18_REG_INCLUDE_CONFIG); - - /* Performance counters with SQTT are considered deprecated. */ - uint32_t token_exclude = V_008D18_TOKEN_EXCLUDE_PERF; - - if (!device->sqtt.instruction_timing_enabled) { - /* Reduce SQTT traffic when instruction timing isn't enabled. */ - token_exclude |= V_008D18_TOKEN_EXCLUDE_VMEMEXEC | V_008D18_TOKEN_EXCLUDE_ALUEXEC | - V_008D18_TOKEN_EXCLUDE_VALUINST | V_008D18_TOKEN_EXCLUDE_IMMEDIATE | - V_008D18_TOKEN_EXCLUDE_INST; - } - sqtt_token_mask |= - S_008D18_TOKEN_EXCLUDE(token_exclude) | S_008D18_BOP_EVENTS_TOKEN_INCLUDE(gfx_level == GFX10_3); - - radeon_set_privileged_config_reg(cs, R_008D18_SQ_THREAD_TRACE_TOKEN_MASK, sqtt_token_mask); - - /* Should be emitted last (it enables thread traces). */ - radeon_set_privileged_config_reg(cs, R_008D1C_SQ_THREAD_TRACE_CTRL, ac_sqtt_get_ctrl(&pdev->info, true)); - } else { - /* Order seems important for the following 4 registers. */ - radeon_set_uconfig_reg(cs, R_030CDC_SQ_THREAD_TRACE_BASE2, S_030CDC_ADDR_HI(shifted_va >> 32)); - - radeon_set_uconfig_reg_seq(cs, R_030CC0_SQ_THREAD_TRACE_BASE, 2); - radeon_emit(cs, shifted_va); - radeon_emit(cs, S_030CC4_SIZE(shifted_size)); - - radeon_set_uconfig_reg(cs, R_030CD4_SQ_THREAD_TRACE_CTRL, S_030CD4_RESET_BUFFER(1)); - - uint32_t sqtt_mask = S_030CC8_CU_SEL(active_cu) | S_030CC8_SH_SEL(0) | S_030CC8_SIMD_EN(0xf) | - S_030CC8_VM_ID_MASK(0) | S_030CC8_REG_STALL_EN(1) | S_030CC8_SPI_STALL_EN(1) | - S_030CC8_SQ_STALL_EN(1); - - if (pdev->info.gfx_level < GFX9) { - sqtt_mask |= S_030CC8_RANDOM_SEED(0xffff); - } - - radeon_set_uconfig_reg_seq(cs, R_030CC8_SQ_THREAD_TRACE_MASK, 3); - radeon_emit(cs, sqtt_mask); - /* Trace all tokens and registers. */ - radeon_emit(cs, S_030CCC_TOKEN_MASK(0xbfff) | S_030CCC_REG_MASK(0xff) | S_030CCC_REG_DROP_ON_STALL(0)); - /* Enable SQTT perf counters for all CUs. */ - radeon_emit(cs, S_030CD0_SH0_MASK(0xffff) | S_030CD0_SH1_MASK(0xffff)); - - radeon_set_uconfig_reg(cs, R_030CE0_SQ_THREAD_TRACE_TOKEN_MASK2, 0xffffffff); - - radeon_set_uconfig_reg(cs, R_030CEC_SQ_THREAD_TRACE_HIWATER, S_030CEC_HIWATER(4)); - - if (pdev->info.gfx_level == GFX9) { - /* Reset thread trace status errors. */ - radeon_set_uconfig_reg(cs, R_030CE8_SQ_THREAD_TRACE_STATUS, S_030CE8_UTC_ERROR(0)); - } - - /* Enable the thread trace mode. */ - uint32_t sqtt_mode = S_030CD8_MASK_PS(1) | S_030CD8_MASK_VS(1) | S_030CD8_MASK_GS(1) | S_030CD8_MASK_ES(1) | - S_030CD8_MASK_HS(1) | S_030CD8_MASK_LS(1) | S_030CD8_MASK_CS(1) | - S_030CD8_AUTOFLUSH_EN(1) | /* periodically flush SQTT data to memory */ - S_030CD8_MODE(1); - - if (pdev->info.gfx_level == GFX9) { - /* Count SQTT traffic in TCC perf counters. */ - sqtt_mode |= S_030CD8_TC_PERF_EN(1); - } - - radeon_set_uconfig_reg(cs, R_030CD8_SQ_THREAD_TRACE_MODE, sqtt_mode); - } - } - - /* Restore global broadcasting. */ - radeon_set_uconfig_reg( - cs, R_030800_GRBM_GFX_INDEX, - S_030800_SE_BROADCAST_WRITES(1) | S_030800_SH_BROADCAST_WRITES(1) | S_030800_INSTANCE_BROADCAST_WRITES(1)); - - /* Start the thread trace with a different event based on the queue. */ - if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B878_COMPUTE_THREAD_TRACE_ENABLE, S_00B878_THREAD_TRACE_ENABLE(1)); - } else { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_THREAD_TRACE_START) | EVENT_INDEX(0)); - } -} - -static const uint32_t gfx8_sqtt_info_regs[] = { - R_030CE4_SQ_THREAD_TRACE_WPTR, - R_030CE8_SQ_THREAD_TRACE_STATUS, - R_008E40_SQ_THREAD_TRACE_CNTR, -}; - -static const uint32_t gfx9_sqtt_info_regs[] = { - R_030CE4_SQ_THREAD_TRACE_WPTR, - R_030CE8_SQ_THREAD_TRACE_STATUS, - R_030CF0_SQ_THREAD_TRACE_CNTR, -}; - -static const uint32_t gfx10_sqtt_info_regs[] = { - R_008D10_SQ_THREAD_TRACE_WPTR, - R_008D20_SQ_THREAD_TRACE_STATUS, - R_008D24_SQ_THREAD_TRACE_DROPPED_CNTR, -}; - -static const uint32_t gfx11_sqtt_info_regs[] = { - R_0367BC_SQ_THREAD_TRACE_WPTR, - R_0367D0_SQ_THREAD_TRACE_STATUS, - R_0367E8_SQ_THREAD_TRACE_DROPPED_CNTR, -}; -static void -radv_copy_sqtt_info_regs(const struct radv_device *device, struct radeon_cmdbuf *cs, unsigned se_index) -{ - const struct radv_physical_device *pdev = radv_device_physical(device); - const uint32_t *sqtt_info_regs = NULL; - - if (pdev->info.gfx_level >= GFX11) { - sqtt_info_regs = gfx11_sqtt_info_regs; - } else if (pdev->info.gfx_level >= GFX10) { - sqtt_info_regs = gfx10_sqtt_info_regs; - } else if (pdev->info.gfx_level == GFX9) { - sqtt_info_regs = gfx9_sqtt_info_regs; - } else { - assert(pdev->info.gfx_level == GFX8); - sqtt_info_regs = gfx8_sqtt_info_regs; - } - - /* Get the VA where the info struct is stored for this SE. */ - uint64_t info_va = ac_sqtt_get_info_va(device->sqtt.buffer_va, se_index); - - /* Copy back the info struct one DWORD at a time. */ - for (unsigned i = 0; i < 3; i++) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, sqtt_info_regs[i] >> 2); - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, (info_va + i * 4)); - radeon_emit(cs, (info_va + i * 4) >> 32); - } - - if (pdev->info.gfx_level == GFX11) { - /* On GFX11, SQ_THREAD_TRACE_WPTR is incremented from the "initial WPTR address" instead of 0. - * To get the number of bytes (in units of 32 bytes) written by SQTT, the workaround is to - * subtract SQ_THREAD_TRACE_WPTR from the "initial WPTR address" as follow: - * - * 1) get the current buffer base address for this SE - * 2) shift right by 5 bits because SQ_THREAD_TRACE_WPTR is 32-byte aligned - * 3) mask off the higher 3 bits because WPTR.OFFSET is 29 bits - */ - uint64_t data_va = ac_sqtt_get_data_va(&pdev->info, &device->sqtt, se_index); - uint64_t shifted_data_va = (data_va >> 5); - uint32_t init_wptr_value = shifted_data_va & 0x1fffffff; - - radeon_emit(cs, PKT3(PKT3_ATOMIC_MEM, 7, 0)); - radeon_emit(cs, ATOMIC_OP(TC_OP_ATOMIC_SUB_32)); - radeon_emit(cs, info_va); /* addr lo */ - radeon_emit(cs, info_va >> 32); /* addr hi */ - radeon_emit(cs, init_wptr_value); /* data lo */ - radeon_emit(cs, 0); /* data hi */ - radeon_emit(cs, 0); /* compare data lo */ - radeon_emit(cs, 0); /* compare data hi */ - radeon_emit(cs, 0); /* loop interval */ - } + ac_pm4_free_state(pm4); } static void radv_emit_sqtt_stop(const struct radv_device *device, struct radeon_cmdbuf *cs, enum radv_queue_family qf) { const struct radv_physical_device *pdev = radv_device_physical(device); - const enum amd_gfx_level gfx_level = pdev->info.gfx_level; - unsigned max_se = pdev->info.max_se; + const bool is_compute_queue = qf == RADV_QUEUE_COMPUTE; + struct ac_pm4_state *pm4; - radeon_check_space(device->ws, cs, 8 + max_se * 64); + pm4 = ac_pm4_create_sized(&pdev->info, false, 512, is_compute_queue); + if (!pm4) + return; - /* Stop the thread trace with a different event based on the queue. */ - if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B878_COMPUTE_THREAD_TRACE_ENABLE, S_00B878_THREAD_TRACE_ENABLE(0)); - } else { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_THREAD_TRACE_STOP) | EVENT_INDEX(0)); - } + ac_sqtt_emit_stop(&pdev->info, pm4, is_compute_queue); + ac_pm4_finalize(pm4); - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_THREAD_TRACE_FINISH) | EVENT_INDEX(0)); + radeon_check_space(device->ws, cs, pm4->ndw); + radeon_emit_array(cs, pm4->pm4, pm4->ndw); + + ac_pm4_clear_state(pm4, &pdev->info, false, is_compute_queue); if (pdev->info.has_sqtt_rb_harvest_bug) { /* Some chips with disabled RBs should wait for idle because FINISH_DONE doesn't work. */ radv_emit_wait_for_idle(device, cs, qf); } - for (unsigned se = 0; se < max_se; se++) { - if (ac_sqtt_se_is_disabled(&pdev->info, se)) - continue; + ac_sqtt_emit_wait(&pdev->info, pm4, &device->sqtt, is_compute_queue); + ac_pm4_finalize(pm4); - /* Target SEi and SH0. */ - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, - S_030800_SE_INDEX(se) | S_030800_SH_INDEX(0) | S_030800_INSTANCE_BROADCAST_WRITES(1)); + radeon_check_space(device->ws, cs, pm4->ndw); + radeon_emit_array(cs, pm4->pm4, pm4->ndw); - if (pdev->info.gfx_level >= GFX11) { - /* Make sure to wait for the trace buffer. */ - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); - radeon_emit(cs, WAIT_REG_MEM_NOT_EQUAL); /* wait until the register is equal to the reference value */ - radeon_emit(cs, R_0367D0_SQ_THREAD_TRACE_STATUS >> 2); /* register */ - radeon_emit(cs, 0); - radeon_emit(cs, 0); /* reference value */ - radeon_emit(cs, ~C_0367D0_FINISH_DONE); - radeon_emit(cs, 4); /* poll interval */ - - /* Disable the thread trace mode. */ - radeon_set_uconfig_perfctr_reg(gfx_level, qf, cs, R_0367B0_SQ_THREAD_TRACE_CTRL, - ac_sqtt_get_ctrl(&pdev->info, false)); - - /* Wait for thread trace completion. */ - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); - radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ - radeon_emit(cs, R_0367D0_SQ_THREAD_TRACE_STATUS >> 2); /* register */ - radeon_emit(cs, 0); - radeon_emit(cs, 0); /* reference value */ - radeon_emit(cs, ~C_0367D0_BUSY); /* mask */ - radeon_emit(cs, 4); /* poll interval */ - } else if (pdev->info.gfx_level >= GFX10) { - if (!pdev->info.has_sqtt_rb_harvest_bug) { - /* Make sure to wait for the trace buffer. */ - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); - radeon_emit(cs, WAIT_REG_MEM_NOT_EQUAL); /* wait until the register is equal to the reference value */ - radeon_emit(cs, R_008D20_SQ_THREAD_TRACE_STATUS >> 2); /* register */ - radeon_emit(cs, 0); - radeon_emit(cs, 0); /* reference value */ - radeon_emit(cs, ~C_008D20_FINISH_DONE); - radeon_emit(cs, 4); /* poll interval */ - } - - /* Disable the thread trace mode. */ - radeon_set_privileged_config_reg(cs, R_008D1C_SQ_THREAD_TRACE_CTRL, ac_sqtt_get_ctrl(&pdev->info, false)); - - /* Wait for thread trace completion. */ - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); - radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ - radeon_emit(cs, R_008D20_SQ_THREAD_TRACE_STATUS >> 2); /* register */ - radeon_emit(cs, 0); - radeon_emit(cs, 0); /* reference value */ - radeon_emit(cs, ~C_008D20_BUSY); /* mask */ - radeon_emit(cs, 4); /* poll interval */ - } else { - /* Disable the thread trace mode. */ - radeon_set_uconfig_reg(cs, R_030CD8_SQ_THREAD_TRACE_MODE, S_030CD8_MODE(0)); - - /* Wait for thread trace completion. */ - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); - radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ - radeon_emit(cs, R_030CE8_SQ_THREAD_TRACE_STATUS >> 2); /* register */ - radeon_emit(cs, 0); - radeon_emit(cs, 0); /* reference value */ - radeon_emit(cs, ~C_030CE8_BUSY); /* mask */ - radeon_emit(cs, 4); /* poll interval */ - } - - radv_copy_sqtt_info_regs(device, cs, se); - } - - /* Restore global broadcasting. */ - radeon_set_uconfig_reg( - cs, R_030800_GRBM_GFX_INDEX, - S_030800_SE_BROADCAST_WRITES(1) | S_030800_SH_BROADCAST_WRITES(1) | S_030800_INSTANCE_BROADCAST_WRITES(1)); + ac_pm4_free_state(pm4); } void