diff --git a/src/broadcom/compiler/v3d_compiler.h b/src/broadcom/compiler/v3d_compiler.h index cf4ff5d6277..b3dae552201 100644 --- a/src/broadcom/compiler/v3d_compiler.h +++ b/src/broadcom/compiler/v3d_compiler.h @@ -956,6 +956,25 @@ struct v3d_compute_prog_data { bool has_subgroups; }; +struct vpm_config { + uint32_t As; + uint32_t Vc; + uint32_t Gs; + uint32_t Gd; + uint32_t Gv; + uint32_t Ve; + uint32_t gs_width; +}; + +bool +v3d_compute_vpm_config(struct v3d_device_info *devinfo, + struct v3d_vs_prog_data *vs_bin, + struct v3d_vs_prog_data *vs, + struct v3d_gs_prog_data *gs_bin, + struct v3d_gs_prog_data *gs, + struct vpm_config *vpm_cfg_bin, + struct vpm_config *vpm_cfg); + static inline bool vir_has_uniform(struct qinst *inst) { diff --git a/src/broadcom/compiler/vir.c b/src/broadcom/compiler/vir.c index 02ce98df6f6..990648b61a6 100644 --- a/src/broadcom/compiler/vir.c +++ b/src/broadcom/compiler/vir.c @@ -1963,3 +1963,174 @@ vir_get_stage_name(struct v3d_compile *c) else return gl_shader_stage_name(c->s->info.stage); } + +static inline uint32_t +compute_vpm_size_in_sectors(const struct v3d_device_info *devinfo) +{ + assert(devinfo->vpm_size > 0); + const uint32_t sector_size = V3D_CHANNELS * sizeof(uint32_t) * 8; + return devinfo->vpm_size / sector_size; +} + +/* Computes various parameters affecting VPM memory configuration for programs + * involving geometry shaders to ensure the program fits in memory and honors + * requirements described in section "VPM usage" of the programming manual. + */ +static bool +compute_vpm_config_gs(struct v3d_device_info *devinfo, + struct v3d_vs_prog_data *vs, + struct v3d_gs_prog_data *gs, + struct vpm_config *vpm_cfg_out) +{ + const uint32_t A = vs->separate_segments ? 1 : 0; + const uint32_t Ad = vs->vpm_input_size; + const uint32_t Vd = vs->vpm_output_size; + + const uint32_t vpm_size = compute_vpm_size_in_sectors(devinfo); + + /* Try to fit program into our VPM memory budget by adjusting + * configurable parameters iteratively. We do this in two phases: + * the first phase tries to fit the program into the total available + * VPM memory. If we succeed at that, then the second phase attempts + * to fit the program into half of that budget so we can run bin and + * render programs in parallel. + */ + struct vpm_config vpm_cfg[2]; + struct vpm_config *final_vpm_cfg = NULL; + uint32_t phase = 0; + + vpm_cfg[phase].As = 1; + vpm_cfg[phase].Gs = 1; + vpm_cfg[phase].Gd = gs->vpm_output_size; + vpm_cfg[phase].gs_width = gs->simd_width; + + /* While there is a requirement that Vc >= [Vn / 16], this is + * always the case when tessellation is not present because in that + * case Vn can only be 6 at most (when input primitive is triangles + * with adjacency). + * + * We always choose Vc=2. We can't go lower than this due to GFXH-1744, + * and Broadcom has not found it worth it to increase it beyond this + * in general. Increasing Vc also increases VPM memory pressure which + * can turn up being detrimental for performance in some scenarios. + */ + vpm_cfg[phase].Vc = 2; + + /* Gv is a constraint on the hardware to not exceed the + * specified number of vertex segments per GS batch. If adding a + * new primitive to a GS batch would result in a range of more + * than Gv vertex segments being referenced by the batch, then + * the hardware will flush the batch and start a new one. This + * means that we can choose any value we want, we just need to + * be aware that larger values improve GS batch utilization + * at the expense of more VPM memory pressure (which can affect + * other performance aspects, such as GS dispatch width). + * We start with the largest value, and will reduce it if we + * find that total memory pressure is too high. + */ + vpm_cfg[phase].Gv = 3; + do { + /* When GS is present in absence of TES, then we need to satisfy + * that Ve >= Gv. We go with the smallest value of Ve to avoid + * increasing memory pressure. + */ + vpm_cfg[phase].Ve = vpm_cfg[phase].Gv; + + uint32_t vpm_sectors = + A * vpm_cfg[phase].As * Ad + + (vpm_cfg[phase].Vc + vpm_cfg[phase].Ve) * Vd + + vpm_cfg[phase].Gs * vpm_cfg[phase].Gd; + + /* Ideally we want to use no more than half of the available + * memory so we can execute a bin and render program in parallel + * without stalls. If we achieved that then we are done. + */ + if (vpm_sectors <= vpm_size / 2) { + final_vpm_cfg = &vpm_cfg[phase]; + break; + } + + /* At the very least, we should not allocate more than the + * total available VPM memory. If we have a configuration that + * succeeds at this we save it and continue to see if we can + * meet the half-memory-use criteria too. + */ + if (phase == 0 && vpm_sectors <= vpm_size) { + vpm_cfg[1] = vpm_cfg[0]; + phase = 1; + } + + /* Try lowering Gv */ + if (vpm_cfg[phase].Gv > 0) { + vpm_cfg[phase].Gv--; + continue; + } + + /* Try lowering GS dispatch width */ + if (vpm_cfg[phase].gs_width > 1) { + do { + vpm_cfg[phase].gs_width >>= 1; + vpm_cfg[phase].Gd = align(vpm_cfg[phase].Gd, 2) / 2; + } while (vpm_cfg[phase].gs_width == 2); + + /* Reset Gv to max after dropping dispatch width */ + vpm_cfg[phase].Gv = 3; + continue; + } + + /* We ran out of options to reduce memory pressure. If we + * are at phase 1 we have at least a valid configuration, so we + * we use that. + */ + if (phase == 1) + final_vpm_cfg = &vpm_cfg[0]; + break; + } while (true); + + if (!final_vpm_cfg) + return false; + + assert(final_vpm_cfg); + assert(final_vpm_cfg->Gd <= 16); + assert(final_vpm_cfg->Gv < 4); + assert(final_vpm_cfg->Ve < 4); + assert(final_vpm_cfg->Vc >= 2 && final_vpm_cfg->Vc <= 4); + assert(final_vpm_cfg->gs_width == 1 || + final_vpm_cfg->gs_width == 4 || + final_vpm_cfg->gs_width == 8 || + final_vpm_cfg->gs_width == 16); + + *vpm_cfg_out = *final_vpm_cfg; + return true; +} + +bool +v3d_compute_vpm_config(struct v3d_device_info *devinfo, + struct v3d_vs_prog_data *vs_bin, + struct v3d_vs_prog_data *vs, + struct v3d_gs_prog_data *gs_bin, + struct v3d_gs_prog_data *gs, + struct vpm_config *vpm_cfg_bin, + struct vpm_config *vpm_cfg) +{ + assert(vs && vs_bin); + assert((gs != NULL) == (gs_bin != NULL)); + + if (!gs) { + vpm_cfg_bin->As = 1; + vpm_cfg_bin->Ve = 0; + vpm_cfg_bin->Vc = vs_bin->vcm_cache_size; + + vpm_cfg->As = 1; + vpm_cfg->Ve = 0; + vpm_cfg->Vc = vs->vcm_cache_size; + } else { + if (!compute_vpm_config_gs(devinfo, vs_bin, gs_bin, vpm_cfg_bin)) + return false; + + if (!compute_vpm_config_gs(devinfo, vs, gs, vpm_cfg)) + return false; + } + + return true; +} diff --git a/src/broadcom/vulkan/v3dv_pipeline.c b/src/broadcom/vulkan/v3dv_pipeline.c index 2fd7f0c457e..fe69f6d0e5e 100644 --- a/src/broadcom/vulkan/v3dv_pipeline.c +++ b/src/broadcom/vulkan/v3dv_pipeline.c @@ -2355,148 +2355,6 @@ pipeline_compile_graphics(struct v3dv_pipeline *pipeline, return compute_vpm_config(pipeline); } -static inline uint32_t -compute_vpm_size_in_sectors(const struct v3d_device_info *devinfo) -{ - assert(devinfo->vpm_size > 0); - const uint32_t sector_size = V3D_CHANNELS * sizeof(uint32_t) * 8; - return devinfo->vpm_size / sector_size; -} - -/* Computes various parameters affecting VPM memory configuration for programs - * involving geometry shaders to ensure the program fits in memory and honors - * requirements described in section "VPM usage" of the programming manual. - * - * FIXME: put this code in common and share with v3d. - */ -static bool -compute_vpm_config_gs(struct v3d_device_info *devinfo, - struct v3d_vs_prog_data *vs, - struct v3d_gs_prog_data *gs, - struct vpm_config *vpm_cfg_out) -{ - const uint32_t A = vs->separate_segments ? 1 : 0; - const uint32_t Ad = vs->vpm_input_size; - const uint32_t Vd = vs->vpm_output_size; - - const uint32_t vpm_size = compute_vpm_size_in_sectors(devinfo); - - /* Try to fit program into our VPM memory budget by adjusting - * configurable parameters iteratively. We do this in two phases: - * the first phase tries to fit the program into the total available - * VPM memory. If we succeed at that, then the second phase attempts - * to fit the program into half of that budget so we can run bin and - * render programs in parallel. - */ - struct vpm_config vpm_cfg[2]; - struct vpm_config *final_vpm_cfg = NULL; - uint32_t phase = 0; - - vpm_cfg[phase].As = 1; - vpm_cfg[phase].Gs = 1; - vpm_cfg[phase].Gd = gs->vpm_output_size; - vpm_cfg[phase].gs_width = gs->simd_width; - - /* While there is a requirement that Vc >= [Vn / 16], this is - * always the case when tessellation is not present because in that - * case Vn can only be 6 at most (when input primitive is triangles - * with adjacency). - * - * We always choose Vc=2. We can't go lower than this due to GFXH-1744, - * and Broadcom has not found it worth it to increase it beyond this - * in general. Increasing Vc also increases VPM memory pressure which - * can turn up being detrimental for performance in some scenarios. - */ - vpm_cfg[phase].Vc = 2; - - /* Gv is a constraint on the hardware to not exceed the - * specified number of vertex segments per GS batch. If adding a - * new primitive to a GS batch would result in a range of more - * than Gv vertex segments being referenced by the batch, then - * the hardware will flush the batch and start a new one. This - * means that we can choose any value we want, we just need to - * be aware that larger values improve GS batch utilization - * at the expense of more VPM memory pressure (which can affect - * other performance aspects, such as GS dispatch width). - * We start with the largest value, and will reduce it if we - * find that total memory pressure is too high. - */ - vpm_cfg[phase].Gv = 3; - do { - /* When GS is present in absence of TES, then we need to satisfy - * that Ve >= Gv. We go with the smallest value of Ve to avoid - * increasing memory pressure. - */ - vpm_cfg[phase].Ve = vpm_cfg[phase].Gv; - - uint32_t vpm_sectors = - A * vpm_cfg[phase].As * Ad + - (vpm_cfg[phase].Vc + vpm_cfg[phase].Ve) * Vd + - vpm_cfg[phase].Gs * vpm_cfg[phase].Gd; - - /* Ideally we want to use no more than half of the available - * memory so we can execute a bin and render program in parallel - * without stalls. If we achieved that then we are done. - */ - if (vpm_sectors <= vpm_size / 2) { - final_vpm_cfg = &vpm_cfg[phase]; - break; - } - - /* At the very least, we should not allocate more than the - * total available VPM memory. If we have a configuration that - * succeeds at this we save it and continue to see if we can - * meet the half-memory-use criteria too. - */ - if (phase == 0 && vpm_sectors <= vpm_size) { - vpm_cfg[1] = vpm_cfg[0]; - phase = 1; - } - - /* Try lowering Gv */ - if (vpm_cfg[phase].Gv > 0) { - vpm_cfg[phase].Gv--; - continue; - } - - /* Try lowering GS dispatch width */ - if (vpm_cfg[phase].gs_width > 1) { - do { - vpm_cfg[phase].gs_width >>= 1; - vpm_cfg[phase].Gd = align(vpm_cfg[phase].Gd, 2) / 2; - } while (vpm_cfg[phase].gs_width == 2); - - /* Reset Gv to max after dropping dispatch width */ - vpm_cfg[phase].Gv = 3; - continue; - } - - /* We ran out of options to reduce memory pressure. If we - * are at phase 1 we have at least a valid configuration, so we - * we use that. - */ - if (phase == 1) - final_vpm_cfg = &vpm_cfg[0]; - break; - } while (true); - - if (!final_vpm_cfg) - return false; - - assert(final_vpm_cfg); - assert(final_vpm_cfg->Gd <= 16); - assert(final_vpm_cfg->Gv < 4); - assert(final_vpm_cfg->Ve < 4); - assert(final_vpm_cfg->Vc >= 2 && final_vpm_cfg->Vc <= 4); - assert(final_vpm_cfg->gs_width == 1 || - final_vpm_cfg->gs_width == 4 || - final_vpm_cfg->gs_width == 8 || - final_vpm_cfg->gs_width == 16); - - *vpm_cfg_out = *final_vpm_cfg; - return true; -} - static VkResult compute_vpm_config(struct v3dv_pipeline *pipeline) { @@ -2507,31 +2365,22 @@ compute_vpm_config(struct v3dv_pipeline *pipeline) struct v3d_vs_prog_data *vs = vs_variant->prog_data.vs; struct v3d_vs_prog_data *vs_bin =vs_bin_variant->prog_data.vs; - if (!pipeline->has_gs) { - pipeline->vpm_cfg_bin.As = 1; - pipeline->vpm_cfg_bin.Ve = 0; - pipeline->vpm_cfg_bin.Vc = vs_bin->vcm_cache_size; - - pipeline->vpm_cfg.As = 1; - pipeline->vpm_cfg.Ve = 0; - pipeline->vpm_cfg.Vc = vs->vcm_cache_size; - } else { + struct v3d_gs_prog_data *gs = NULL; + struct v3d_gs_prog_data *gs_bin = NULL; + if (pipeline->has_gs) { struct v3dv_shader_variant *gs_variant = pipeline->shared_data->variants[BROADCOM_SHADER_GEOMETRY]; struct v3dv_shader_variant *gs_bin_variant = pipeline->shared_data->variants[BROADCOM_SHADER_GEOMETRY_BIN]; - struct v3d_gs_prog_data *gs = gs_variant->prog_data.gs; - struct v3d_gs_prog_data *gs_bin = gs_bin_variant->prog_data.gs; + gs = gs_variant->prog_data.gs; + gs_bin = gs_bin_variant->prog_data.gs; + } - if (!compute_vpm_config_gs(&pipeline->device->devinfo, - vs_bin, gs_bin, &pipeline->vpm_cfg_bin)) { - return VK_ERROR_OUT_OF_DEVICE_MEMORY; - } - - if (!compute_vpm_config_gs(&pipeline->device->devinfo, - vs, gs, &pipeline->vpm_cfg)) { - return VK_ERROR_OUT_OF_DEVICE_MEMORY; - } + if (!v3d_compute_vpm_config(&pipeline->device->devinfo, + vs_bin, vs, gs_bin, gs, + &pipeline->vpm_cfg_bin, + &pipeline->vpm_cfg)) { + return VK_ERROR_OUT_OF_DEVICE_MEMORY; } return VK_SUCCESS; diff --git a/src/broadcom/vulkan/v3dv_private.h b/src/broadcom/vulkan/v3dv_private.h index ca28f111884..6440695246d 100644 --- a/src/broadcom/vulkan/v3dv_private.h +++ b/src/broadcom/vulkan/v3dv_private.h @@ -1447,20 +1447,6 @@ struct v3dv_pipeline_stage { uint32_t program_id; }; -/* FIXME: although the full vpm_config is not required at this point, as we - * don't plan to initially support GS, it is more readable and serves as a - * placeholder, to have the struct and fill it with default values. - */ -struct vpm_config { - uint32_t As; - uint32_t Vc; - uint32_t Gs; - uint32_t Gd; - uint32_t Gv; - uint32_t Ve; - uint32_t gs_width; -}; - /* We are using the descriptor pool entry for two things: * * Track the allocated sets, so we can properly free it if needed * * Track the suballocated pool bo regions, so if some descriptor set is diff --git a/src/gallium/drivers/v3d/v3dx_draw.c b/src/gallium/drivers/v3d/v3dx_draw.c index 91a1048a8b9..b9b5254c9f8 100644 --- a/src/gallium/drivers/v3d/v3dx_draw.c +++ b/src/gallium/drivers/v3d/v3dx_draw.c @@ -358,16 +358,6 @@ v3d_emit_wait_for_tf_if_needed(struct v3d_context *v3d, struct v3d_job *job) } } -struct vpm_config { - uint32_t As; - uint32_t Vc; - uint32_t Gs; - uint32_t Gd; - uint32_t Gv; - uint32_t Ve; - uint32_t gs_width; -}; - #if V3D_VERSION >= 41 static void v3d_emit_gs_state_record(struct v3d_job *job, @@ -484,151 +474,6 @@ v3d_emit_tes_gs_shader_params(struct v3d_job *job, shader.gbg_min_gs_output_segments_required_in_play = 1; } } - -static inline uint32_t -compute_vpm_size_in_sectors(const struct v3d_device_info *devinfo) -{ - assert(devinfo->vpm_size > 0); - const uint32_t sector_size = V3D_CHANNELS * sizeof(uint32_t) * 8; - return devinfo->vpm_size / sector_size; -} - -/* Computes various parameters affecting VPM memory configuration for programs - * involving geometry shaders to ensure the program fits in memory and honors - * requirements described in section "VPM usage" of the programming manual. - */ -static void -compute_vpm_config_gs(struct v3d_device_info *devinfo, - struct v3d_vs_prog_data *vs, - struct v3d_gs_prog_data *gs, - struct vpm_config *vpm_cfg_out) -{ - const uint32_t A = vs->separate_segments ? 1 : 0; - const uint32_t Ad = vs->vpm_input_size; - const uint32_t Vd = vs->vpm_output_size; - - const uint32_t vpm_size = compute_vpm_size_in_sectors(devinfo); - - /* Try to fit program into our VPM memory budget by adjusting - * configurable parameters iteratively. We do this in two phases: - * the first phase tries to fit the program into the total available - * VPM memory. If we succeed at that, then the second phase attempts - * to fit the program into half of that budget so we can run bin and - * render programs in parallel. - */ - struct vpm_config vpm_cfg[2]; - struct vpm_config *final_vpm_cfg = NULL; - uint32_t phase = 0; - - vpm_cfg[phase].As = 1; - vpm_cfg[phase].Gs = 1; - vpm_cfg[phase].Gd = gs->vpm_output_size; - vpm_cfg[phase].gs_width = gs->simd_width; - - /* While there is a requirement that Vc >= [Vn / 16], this is - * always the case when tessellation is not present because in that - * case Vn can only be 6 at most (when input primitive is triangles - * with adjacency). - * - * We always choose Vc=2. We can't go lower than this due to GFXH-1744, - * and Broadcom has not found it worth it to increase it beyond this - * in general. Increasing Vc also increases VPM memory pressure which - * can turn up being detrimental for performance in some scenarios. - */ - vpm_cfg[phase].Vc = 2; - - /* Gv is a constraint on the hardware to not exceed the - * specified number of vertex segments per GS batch. If adding a - * new primitive to a GS batch would result in a range of more - * than Gv vertex segments being referenced by the batch, then - * the hardware will flush the batch and start a new one. This - * means that we can choose any value we want, we just need to - * be aware that larger values improve GS batch utilization - * at the expense of more VPM memory pressure (which can affect - * other performance aspects, such as GS dispatch width). - * We start with the largest value, and will reduce it if we - * find that total memory pressure is too high. - */ - vpm_cfg[phase].Gv = 3; - do { - /* When GS is present in absence of TES, then we need to satisfy - * that Ve >= Gv. We go with the smallest value of Ve to avoid - * increasing memory pressure. - */ - vpm_cfg[phase].Ve = vpm_cfg[phase].Gv; - - uint32_t vpm_sectors = - A * vpm_cfg[phase].As * Ad + - (vpm_cfg[phase].Vc + vpm_cfg[phase].Ve) * Vd + - vpm_cfg[phase].Gs * vpm_cfg[phase].Gd; - - /* Ideally we want to use no more than half of the available - * memory so we can execute a bin and render program in parallel - * without stalls. If we achieved that then we are done. - */ - if (vpm_sectors <= vpm_size / 2) { - final_vpm_cfg = &vpm_cfg[phase]; - break; - } - - /* At the very least, we should not allocate more than the - * total available VPM memory. If we have a configuration that - * succeeds at this we save it and continue to see if we can - * meet the half-memory-use criteria too. - */ - if (phase == 0 && vpm_sectors <= vpm_size) { - vpm_cfg[1] = vpm_cfg[0]; - phase = 1; - } - - /* Try lowering Gv */ - if (vpm_cfg[phase].Gv > 0) { - vpm_cfg[phase].Gv--; - continue; - } - - /* Try lowering GS dispatch width */ - if (vpm_cfg[phase].gs_width > 1) { - do { - vpm_cfg[phase].gs_width >>= 1; - vpm_cfg[phase].Gd = - align(vpm_cfg[phase].Gd, 2) / 2; - } while (vpm_cfg[phase].gs_width == 2); - - /* Reset Gv to max after dropping dispatch width */ - vpm_cfg[phase].Gv = 3; - continue; - } - - /* We ran out of options to reduce memory pressure. If we - * are at phase 1 we have at least a valid configuration, so we - * we use that. - */ - if (phase == 1) - final_vpm_cfg = &vpm_cfg[0]; - break; - } while (true); - - if (!final_vpm_cfg) { - /* FIXME: maybe return a boolean to indicate failure and use - * that to stop the submission for this draw call. - */ - fprintf(stderr, "Failed to allocate VPM memory.\n"); - abort(); - } - - assert(final_vpm_cfg); - assert(final_vpm_cfg->Gd <= 16); - assert(final_vpm_cfg->Gv < 4); - assert(final_vpm_cfg->Ve < 4); - assert(final_vpm_cfg->Vc >= 2 && final_vpm_cfg->Vc <= 4); - assert(final_vpm_cfg->gs_width == 1 || - final_vpm_cfg->gs_width == 4 || - final_vpm_cfg->gs_width == 8 || - final_vpm_cfg->gs_width == 16); - - *vpm_cfg_out = *final_vpm_cfg; -} #endif static void @@ -713,43 +558,32 @@ v3d_emit_gl_shader_state(struct v3d_context *v3d, struct vpm_config vpm_cfg_bin, vpm_cfg; assert(v3d->screen->devinfo.ver >= 41 || !v3d->prog.gs); - if (!v3d->prog.gs) { - vpm_cfg_bin.As = 1; - vpm_cfg_bin.Ve = 0; - vpm_cfg_bin.Vc = v3d->prog.cs->prog_data.vs->vcm_cache_size; + v3d_compute_vpm_config(&v3d->screen->devinfo, + v3d->prog.cs->prog_data.vs, + v3d->prog.vs->prog_data.vs, + v3d->prog.gs ? v3d->prog.gs_bin->prog_data.gs : NULL, + v3d->prog.gs ? v3d->prog.gs->prog_data.gs : NULL, + &vpm_cfg_bin, + &vpm_cfg); - vpm_cfg.As = 1; - vpm_cfg.Ve = 0; - vpm_cfg.Vc = v3d->prog.vs->prog_data.vs->vcm_cache_size; - } - else { + if (v3d->prog.gs) { #if V3D_VERSION >= 41 v3d_emit_gs_state_record(v3d->job, v3d->prog.gs_bin, gs_bin_uniforms, v3d->prog.gs, gs_uniforms); struct v3d_gs_prog_data *gs = v3d->prog.gs->prog_data.gs; - struct v3d_gs_prog_data *gs_bin = v3d->prog.gs_bin->prog_data.gs; - v3d_emit_tes_gs_common_params(v3d->job, gs->out_prim_type, gs->num_invocations); /* Bin Tes/Gs params */ - struct v3d_vs_prog_data *vs_bin = v3d->prog.cs->prog_data.vs; - compute_vpm_config_gs(&v3d->screen->devinfo, - vs_bin, gs_bin, &vpm_cfg_bin); - v3d_emit_tes_gs_shader_params(v3d->job, vpm_cfg_bin.gs_width, vpm_cfg_bin.Gd, vpm_cfg_bin.Gv); /* Render Tes/Gs params */ - struct v3d_vs_prog_data *vs = v3d->prog.vs->prog_data.vs; - compute_vpm_config_gs(&v3d->screen->devinfo, - vs, gs, &vpm_cfg); - v3d_emit_tes_gs_shader_params(v3d->job, vpm_cfg.gs_width, vpm_cfg.Gd,