From 5287edd6d2a80e021521818f56f4f92cc4acdc3a Mon Sep 17 00:00:00 2001 From: Simon Perretta Date: Wed, 15 Feb 2023 16:57:24 +0000 Subject: [PATCH] pvr: Register allocation improvements Signed-off-by: Simon Perretta Acked-by: Frank Binns Part-of: --- src/imagination/rogue/passes/rogue_regalloc.c | 194 ++++++++---------- src/imagination/rogue/rogue.h | 6 + src/imagination/rogue/rogue_compile.c | 24 ++- 3 files changed, 116 insertions(+), 108 deletions(-) diff --git a/src/imagination/rogue/passes/rogue_regalloc.c b/src/imagination/rogue/passes/rogue_regalloc.c index d46ef62044f..a709c22b972 100644 --- a/src/imagination/rogue/passes/rogue_regalloc.c +++ b/src/imagination/rogue/passes/rogue_regalloc.c @@ -23,7 +23,6 @@ #include "rogue.h" #include "util/macros.h" -#include "util/u_qsort.h" #include "util/ralloc.h" #include "util/register_allocate.h" @@ -43,18 +42,34 @@ typedef struct rogue_live_range { unsigned end; } rogue_live_range; -static int regarray_cmp(const void *lhs, const void *rhs, UNUSED void *arg) +static void rogue_regarray_liveness(rogue_regarray *regarray, + rogue_live_range *live_range) { - const rogue_regarray *l = lhs; - const rogue_regarray *r = rhs; + assert(list_is_singular(®array->writes) || + list_is_empty(®array->writes)); + if (!list_is_empty(®array->writes)) { + rogue_regarray_write *write = + list_first_entry(®array->writes, rogue_regarray_write, link); + live_range->start = MIN2(live_range->start, write->instr->index); + } - /* Signs swapped for sorting largest->smallest. */ - if (l->size > r->size) - return -1; - else if (l->size < r->size) - return 1; + rogue_foreach_regarray_use (use, regarray) { + live_range->end = MAX2(live_range->end, use->instr->index); + } +} - return 0; +static void rogue_reg_liveness(rogue_reg *reg, rogue_live_range *live_range) +{ + assert(list_is_singular(®->writes) || list_is_empty(®->writes)); + if (!list_is_empty(®->writes)) { + rogue_reg_write *write = + list_first_entry(®->writes, rogue_reg_write, link); + live_range->start = MIN2(live_range->start, write->instr->index); + } + + rogue_foreach_reg_use (use, reg) { + live_range->end = MAX2(live_range->end, use->instr->index); + } } PUBLIC @@ -65,65 +80,72 @@ bool rogue_regalloc(rogue_shader *shader) bool progress = false; - unsigned num_ssa_regs = list_length(&shader->regs[ROGUE_REG_CLASS_SSA]); + unsigned num_ssa_regs = rogue_count_used_regs(shader, ROGUE_REG_CLASS_SSA); if (!num_ssa_regs) return false; - /* Ensure that ssa regs are continuous from zero, and have no gaps. */ - rogue_foreach_reg (reg, shader, ROGUE_REG_CLASS_SSA) { - assert(reg->index < num_ssa_regs); - } - - /* If we already have some temps in use in the shader, we'll skip using them - * for allocation. */ - unsigned num_temp_regs = list_length(&shader->regs[ROGUE_REG_CLASS_TEMP]); + assert(list_is_empty(&shader->regs[ROGUE_REG_CLASS_TEMP])); unsigned hw_temps = rogue_reg_infos[ROGUE_REG_CLASS_TEMP].num; + /* Setup regset and register classes. */ struct ra_regs *ra_regs = ra_alloc_reg_set(shader, hw_temps, true); for (enum rogue_regalloc_class c = 0; c < ROGUE_REGALLOC_CLASS_COUNT; ++c) { ASSERTED struct ra_class *ra_class = ra_alloc_contig_reg_class(ra_regs, regalloc_info[c].stride); assert(c == ra_class_index(ra_class)); - } - for (unsigned t = num_temp_regs; t < hw_temps; ++t) - for (enum rogue_regalloc_class c = 0; c < ROGUE_REGALLOC_CLASS_COUNT; ++c) + for (unsigned t = 0; t < hw_temps; ++t) if (!(t % regalloc_info[c].stride)) - ra_class_add_reg(ra_get_class_from_index(ra_regs, c), t); + ra_class_add_reg(ra_class, t); + } ra_set_finalize(ra_regs, NULL); + /* TODO: Consider tracking this in the shader itself, i.e. one list for child + * regarrays, one for parents. Or, since children are already in a list in + * the parent, only have parent regarrays in the shader. + */ + + /* Count the parent regarrays. */ + unsigned num_parent_regarrays = 0; + rogue_foreach_regarray (regarray, shader) { + if (regarray->parent || regarray->regs[0]->class != ROGUE_REG_CLASS_SSA) + continue; + + ++num_parent_regarrays; + } + + /* Construct list of parent regarrays. */ + rogue_regarray **parent_regarrays = + rzalloc_array_size(ra_regs, + sizeof(*parent_regarrays), + num_parent_regarrays); + + unsigned ra = 0; + rogue_foreach_regarray (regarray, shader) { + if (regarray->parent || regarray->regs[0]->class != ROGUE_REG_CLASS_SSA) + continue; + + parent_regarrays[ra++] = regarray; + } + + /* Prepare live ranges. */ rogue_live_range *ssa_live_range = - rzalloc_array_size(shader, sizeof(*ssa_live_range), num_ssa_regs); + rzalloc_array_size(ra_regs, sizeof(*ssa_live_range), num_ssa_regs); for (unsigned u = 0; u < num_ssa_regs; ++u) ssa_live_range[u].start = ~0U; /* Populate live ranges for register arrays. */ - rogue_foreach_regarray (regarray, shader) { - enum rogue_reg_class class = regarray->regs[0]->class; - if (class != ROGUE_REG_CLASS_SSA) - continue; + for (unsigned u = 0; u < num_parent_regarrays; ++u) { + rogue_regarray *regarray = parent_regarrays[u]; + unsigned base_index = regarray->regs[0]->index; + rogue_live_range *live_range = &ssa_live_range[base_index]; - for (unsigned u = 0; u < regarray->size; ++u) { - rogue_reg *reg = regarray->regs[u]; - rogue_live_range *live_range = &ssa_live_range[reg->index]; + rogue_regarray_liveness(regarray, live_range); - assert(list_is_singular(®array->writes) || - list_is_empty(®array->writes)); - if (!list_is_empty(®array->writes)) { - rogue_regarray_write *write = - list_first_entry(®array->writes, rogue_regarray_write, link); - live_range->start = MIN2(live_range->start, write->instr->index); - } - - rogue_foreach_regarray_use (use, regarray) { - live_range->end = MAX2(live_range->end, use->instr->index); - } - - /* Here dirty represents whether the register has been added to the - * regset yet or not. */ - reg->dirty = false; + rogue_foreach_subarray (subarray, regarray) { + rogue_regarray_liveness(subarray, live_range); } } @@ -133,21 +155,7 @@ bool rogue_regalloc(rogue_shader *shader) continue; rogue_live_range *live_range = &ssa_live_range[reg->index]; - - assert(list_is_singular(®->writes) || list_is_empty(®->writes)); - if (!list_is_empty(®->writes)) { - rogue_reg_write *write = - list_first_entry(®->writes, rogue_reg_write, link); - live_range->start = MIN2(live_range->start, write->instr->index); - } - - rogue_foreach_reg_use (use, reg) { - live_range->end = MAX2(live_range->end, use->instr->index); - } - - /* Here dirty represents whether the register has been added to the regset - * yet or not. */ - reg->dirty = false; + rogue_reg_liveness(reg, live_range); } struct ra_graph *ra_graph = @@ -155,13 +163,9 @@ bool rogue_regalloc(rogue_shader *shader) ralloc_steal(ra_regs, ra_graph); /* Set register class for regarrays/vectors. */ - rogue_foreach_regarray (regarray, shader) { - enum rogue_reg_class class = regarray->regs[0]->class; - if (class != ROGUE_REG_CLASS_SSA) - continue; - - if (regarray->parent) - continue; + for (unsigned u = 0; u < num_parent_regarrays; ++u) { + rogue_regarray *regarray = parent_regarrays[u]; + unsigned base_index = regarray->regs[0]->index; enum rogue_regalloc_class raclass; @@ -173,23 +177,19 @@ bool rogue_regalloc(rogue_shader *shader) unreachable("Unsupported regarray size."); ra_set_node_class(ra_graph, - regarray->regs[0]->index, + base_index, ra_get_class_from_index(ra_regs, raclass)); - - for (unsigned u = 0; u < regarray->size; ++u) - regarray->regs[u]->dirty = true; } /* Set register class for "standalone" registers. */ rogue_foreach_reg (reg, shader, ROGUE_REG_CLASS_SSA) { - if (reg->dirty) + if (reg->regarray) continue; ra_set_node_class(ra_graph, reg->index, ra_get_class_from_index(ra_regs, ROGUE_REGALLOC_CLASS_TEMP_1)); - reg->dirty = true; } /* Build interference graph from overlapping live ranges. */ @@ -213,37 +213,19 @@ bool rogue_regalloc(rogue_shader *shader) if (!ra_allocate(ra_graph)) unreachable("Register allocation failed."); - unsigned regarray_count = list_length(&shader->regarrays); - rogue_regarray **parent_regarrays = - rzalloc_array_size(shader, sizeof(*parent_regarrays), regarray_count); - - /* Construct list of sorted parent regarrays. */ - unsigned num_parent_regarrays = 0; - rogue_foreach_regarray (regarray, shader) { - if (regarray->parent || regarray->regs[0]->class != ROGUE_REG_CLASS_SSA) - continue; - - parent_regarrays[num_parent_regarrays++] = regarray; - } - - util_qsort_r(parent_regarrays, - num_parent_regarrays, - sizeof(*parent_regarrays), - regarray_cmp, - NULL); - + /* Replace SSA regarray registers with allocated physical registers. */ for (unsigned u = 0; u < num_parent_regarrays; ++u) { rogue_regarray *regarray = parent_regarrays[u]; - unsigned start_index = regarray->regs[0]->index; - unsigned new_base_index = ra_get_node_reg(ra_graph, start_index); + unsigned base_index = regarray->regs[0]->index; + unsigned hw_base_index = ra_get_node_reg(ra_graph, base_index); enum rogue_regalloc_class ra_class = - ra_class_index(ra_get_node_class(ra_graph, start_index)); + ra_class_index(ra_get_node_class(ra_graph, base_index)); enum rogue_reg_class new_class = regalloc_info[ra_class].class; bool used = false; for (unsigned r = 0; r < regarray->size; ++r) { - used |= rogue_reg_is_used(shader, new_class, new_base_index + r); + used |= rogue_reg_is_used(shader, new_class, hw_base_index + r); if (used) break; @@ -252,7 +234,7 @@ bool rogue_regalloc(rogue_shader *shader) /* First time using new regarray, modify in place. */ if (!used) { progress |= - rogue_regarray_rewrite(shader, regarray, new_class, new_base_index); + rogue_regarray_rewrite(shader, regarray, new_class, hw_base_index); } else { /* Regarray has already been used, replace references and delete. */ @@ -260,7 +242,7 @@ bool rogue_regalloc(rogue_shader *shader) rogue_regarray *new_regarray = rogue_regarray_cached(shader, regarray->size, new_class, - new_base_index); + hw_base_index); progress |= rogue_regarray_replace(shader, regarray, new_regarray); } } @@ -269,31 +251,31 @@ bool rogue_regalloc(rogue_shader *shader) * registers. */ rogue_foreach_reg_safe (reg, shader, ROGUE_REG_CLASS_SSA) { assert(!reg->regarray); - unsigned new_index = ra_get_node_reg(ra_graph, reg->index); + unsigned hw_index = ra_get_node_reg(ra_graph, reg->index); enum rogue_regalloc_class ra_class = ra_class_index(ra_get_node_class(ra_graph, reg->index)); enum rogue_reg_class new_class = regalloc_info[ra_class].class; /* First time using new register, modify in place. */ - if (!rogue_reg_is_used(shader, new_class, new_index)) { - progress |= rogue_reg_rewrite(shader, reg, new_class, new_index); + if (!rogue_reg_is_used(shader, new_class, hw_index)) { + progress |= rogue_reg_rewrite(shader, reg, new_class, hw_index); } else { /* Register has already been used, replace references and delete. */ assert(list_is_singular(®->writes)); /* SSA reg. */ - rogue_reg *new_reg = rogue_temp_reg(shader, new_index); + rogue_reg *new_reg = rogue_temp_reg(shader, hw_index); progress |= rogue_reg_replace(reg, new_reg); } } - num_temp_regs = list_length(&shader->regs[ROGUE_REG_CLASS_TEMP]); +#ifndef NDEBUG /* Ensure that temp regs are continuous from zero, and have no gaps. */ + unsigned num_temp_regs = list_length(&shader->regs[ROGUE_REG_CLASS_TEMP]); rogue_foreach_reg (reg, shader, ROGUE_REG_CLASS_TEMP) { assert(reg->index < num_temp_regs); } +#endif /* NDEBUG */ - ralloc_free(parent_regarrays); - ralloc_free(ssa_live_range); ralloc_free(ra_regs); return progress; } diff --git a/src/imagination/rogue/rogue.h b/src/imagination/rogue/rogue.h index 0f442f2dd12..46a8e55eedc 100644 --- a/src/imagination/rogue/rogue.h +++ b/src/imagination/rogue/rogue.h @@ -116,6 +116,9 @@ static inline enum reg_bank rogue_reg_bank_encoding(enum rogue_reg_class class) } } +/* TODO: Do this dynamically by iterating + * through regarrays and matching sizes. + */ enum rogue_regalloc_class { ROGUE_REGALLOC_CLASS_TEMP_1, ROGUE_REGALLOC_CLASS_TEMP_2, @@ -2889,6 +2892,9 @@ rogue_build_context_create(rogue_compiler *compiler, void rogue_collect_io_data(rogue_build_ctx *ctx, nir_shader *nir); +unsigned rogue_count_used_regs(const rogue_shader *shader, + enum rogue_reg_class class); + unsigned rogue_coeff_index_fs(rogue_iterator_args *args, gl_varying_slot location, unsigned component); diff --git a/src/imagination/rogue/rogue_compile.c b/src/imagination/rogue/rogue_compile.c index 687557a9911..b4b2f11499d 100644 --- a/src/imagination/rogue/rogue_compile.c +++ b/src/imagination/rogue/rogue_compile.c @@ -357,8 +357,6 @@ trans_nir_intrinsic_load_vulkan_descriptor(rogue_builder *b, flat_desc_idx = pipeline_layout->set_layout[desc_set] ->bindings[u] .descriptor_index; - assert(pipeline_layout->set_layout[desc_set]->bindings[u].type == - desc_type); break; } } @@ -612,6 +610,28 @@ static void trans_nir_alu(rogue_builder *b, nir_alu_instr *alu) unreachable("Unimplemented NIR ALU instruction."); } +PUBLIC +unsigned rogue_count_used_regs(const rogue_shader *shader, + enum rogue_reg_class class) +{ + unsigned reg_count; + if (rogue_reg_infos[class].num) { + reg_count = __bitset_count(shader->regs_used[class], + BITSET_WORDS(rogue_reg_infos[class].num)); + } else { + reg_count = list_length(&shader->regs[class]); + } + +#ifndef NDEBUG + /* Check that registers are contiguous. */ + rogue_foreach_reg (reg, shader, class) { + assert(reg->index < reg_count); + } +#endif /* NDEBUG */ + + return reg_count; +} + static inline void rogue_feedback_used_regs(rogue_build_ctx *ctx, const rogue_shader *shader) {