nir/lower_doubles: Inline functions directly in lower_doubles

Instead of trusting the caller to already have created a softfp64
function shader and added all its functions to our shader, we simply
take the softfp64 shader as an argument and do the function inlining
ouselves.  This means that there's no more nasty functions lying around
that the caller needs to worry about cleaning up.

Reviewed-by: Matt Turner <mattst88@gmail.com>
Reviewed-by: Jordan Justen <jordan.l.justen@intel.com>
Reviewed-by: Kenneth Graunke <kenneth@whitecape.org>

src/compiler/nir/nir.h

@@ -3279,7 +3279,8 @@ nir_lower_int64_options nir_lower_int64_op_to_options_mask(nir_op opcode);
 bool nir_lower_int64(nir_shader *shader, nir_lower_int64_options options);
 
 nir_lower_doubles_options nir_lower_doubles_op_to_options_mask(nir_op opcode);
-bool nir_lower_doubles(nir_shader *shader, nir_lower_doubles_options options);
+bool nir_lower_doubles(nir_shader *shader, const nir_shader *softfp64,
+                       nir_lower_doubles_options options);
 bool nir_lower_pack(nir_shader *shader);
 
 bool nir_normalize_cubemap_coords(nir_shader *shader);

src/compiler/nir/nir_lower_double_ops.c

@@ -426,6 +426,7 @@ lower_mod(nir_builder *b, nir_ssa_def *src0, nir_ssa_def *src1)
 
 static bool
 lower_doubles_instr_to_soft(nir_builder *b, nir_alu_instr *instr,
+                            const nir_shader *softfp64,
                             nir_lower_doubles_options options)
 {
    if (!(options & nir_lower_fp64_full_software))
@@ -553,35 +554,34 @@ lower_doubles_instr_to_soft(nir_builder *b, nir_alu_instr *instr,
       return false;
    }
 
-   nir_shader *shader = b->shader;
    nir_function *func = NULL;
-
+   nir_foreach_function(function, softfp64) {
-   nir_foreach_function(function, shader) {
       if (strcmp(function->name, name) == 0) {
          func = function;
          break;
       }
    }
-   if (!func) {
+   if (!func || !func->impl) {
       fprintf(stderr, "Cannot find function \"%s\"\n", name);
       assert(func);
    }
 
    b->cursor = nir_before_instr(&instr->instr);
 
-   nir_call_instr *call = nir_call_instr_create(shader, func);
+   nir_ssa_def *params[4] = { NULL, };
 
    nir_variable *ret_tmp =
       nir_local_variable_create(b->impl, return_type, "return_tmp");
    nir_deref_instr *ret_deref = nir_build_deref_var(b, ret_tmp);
-   call->params[0] = nir_src_for_ssa(&ret_deref->dest.ssa);
+   params[0] = &ret_deref->dest.ssa;
 
+   assert(nir_op_infos[instr->op].num_inputs + 1 == func->num_params);
    for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
-      nir_src arg = nir_src_for_ssa(nir_imov_alu(b, instr->src[i], 1));
+      assert(i + 1 < ARRAY_SIZE(params));
-      nir_src_copy(&call->params[i + 1], &arg, call);
+      params[i + 1] = nir_imov_alu(b, instr->src[i], 1);
    }
 
-   nir_builder_instr_insert(b, &call->instr);
+   nir_inline_function_impl(b, func->impl, params);
 
    nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
                             nir_src_for_ssa(nir_load_deref(b, ret_deref)));
@@ -608,6 +608,7 @@ nir_lower_doubles_op_to_options_mask(nir_op opcode)
 
 static bool
 lower_doubles_instr(nir_builder *b, nir_alu_instr *instr,
+                    const nir_shader *softfp64,
                     nir_lower_doubles_options options)
 {
    assert(instr->dest.dest.is_ssa);
@@ -621,7 +622,7 @@ lower_doubles_instr(nir_builder *b, nir_alu_instr *instr,
    if (!is_64)
       return false;
 
-   if (lower_doubles_instr_to_soft(b, instr, options))
+   if (lower_doubles_instr_to_soft(b, instr, softfp64, options))
       return true;
 
    if (!(options & nir_lower_doubles_op_to_options_mask(instr->op)))
@@ -677,6 +678,7 @@ lower_doubles_instr(nir_builder *b, nir_alu_instr *instr,
 
 static bool
 nir_lower_doubles_impl(nir_function_impl *impl,
+                       const nir_shader *softfp64,
                        nir_lower_doubles_options options)
 {
    bool progress = false;
@@ -684,17 +686,30 @@ nir_lower_doubles_impl(nir_function_impl *impl,
    nir_builder b;
    nir_builder_init(&b, impl);
 
-   nir_foreach_block(block, impl) {
+   nir_foreach_block_safe(block, impl) {
       nir_foreach_instr_safe(instr, block) {
          if (instr->type == nir_instr_type_alu)
             progress |= lower_doubles_instr(&b, nir_instr_as_alu(instr),
-                                            options);
+                                            softfp64, options);
       }
    }
 
    if (progress) {
-      nir_metadata_preserve(impl, nir_metadata_block_index |
+      if (options & nir_lower_fp64_full_software) {
-                                  nir_metadata_dominance);
+         /* SSA and register indices are completely messed up now */
+         nir_index_ssa_defs(impl);
+         nir_index_local_regs(impl);
+
+         nir_metadata_preserve(impl, nir_metadata_none);
+
+         /* And we have deref casts we need to clean up thanks to function
+          * inlining.
+          */
+         nir_opt_deref_impl(impl);
+      } else {
+         nir_metadata_preserve(impl, nir_metadata_block_index |
+                                     nir_metadata_dominance);
+      }
     } else {
 #ifndef NDEBUG
       impl->valid_metadata &= ~nir_metadata_not_properly_reset;
@@ -705,13 +720,15 @@ nir_lower_doubles_impl(nir_function_impl *impl,
 }
 
 bool
-nir_lower_doubles(nir_shader *shader, nir_lower_doubles_options options)
+nir_lower_doubles(nir_shader *shader,
+                  const nir_shader *softfp64,
+                  nir_lower_doubles_options options)
 {
    bool progress = false;
 
    nir_foreach_function(function, shader) {
       if (function->impl) {
-         progress |= nir_lower_doubles_impl(function->impl, options);
+         progress |= nir_lower_doubles_impl(function->impl, softfp64, options);
       }
    }
 

src/gallium/drivers/iris/iris_program.c

@@ -1384,7 +1384,7 @@ iris_create_uncompiled_shader(struct pipe_context *ctx,
    if (!ish)
       return NULL;
 
-   nir = brw_preprocess_nir(screen->compiler, nir);
+   nir = brw_preprocess_nir(screen->compiler, nir, NULL);
 
    NIR_PASS_V(nir, brw_nir_lower_image_load_store, devinfo);
    NIR_PASS_V(nir, iris_lower_storage_image_derefs);

src/intel/blorp/blorp.c

@@ -192,7 +192,7 @@ blorp_compile_fs(struct blorp_context *blorp, void *mem_ctx,
     */
    wm_prog_data->base.binding_table.texture_start = BLORP_TEXTURE_BT_INDEX;
 
-   nir = brw_preprocess_nir(compiler, nir);
+   nir = brw_preprocess_nir(compiler, nir, NULL);
    nir_remove_dead_variables(nir, nir_var_shader_in);
    nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
 
@@ -221,7 +221,7 @@ blorp_compile_vs(struct blorp_context *blorp, void *mem_ctx,
    nir->options =
       compiler->glsl_compiler_options[MESA_SHADER_VERTEX].NirOptions;
 
-   nir = brw_preprocess_nir(compiler, nir);
+   nir = brw_preprocess_nir(compiler, nir, NULL);
    nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
 
    vs_prog_data->inputs_read = nir->info.inputs_read;

src/intel/compiler/brw_nir.c

@@ -653,7 +653,8 @@ lower_bit_size_callback(const nir_alu_instr *alu, UNUSED void *data)
  * is_scalar = true to scalarize everything prior to code gen.
  */
 nir_shader *
-brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir)
+brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir,
+                   const nir_shader *softfp64)
 {
    const struct gen_device_info *devinfo = compiler->devinfo;
    UNUSED bool progress; /* Written by OPT */
@@ -677,7 +678,7 @@ brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir)
       progress = false;
 
       OPT(nir_lower_int64, nir->options->lower_int64_options);
-      OPT(nir_lower_doubles, nir->options->lower_doubles_options);
+      OPT(nir_lower_doubles, softfp64, nir->options->lower_doubles_options);
 
       /* Necessary to lower add -> sub and div -> mul/rcp */
       OPT(nir_opt_algebraic);
@@ -685,21 +686,6 @@ brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir)
       lowered_64bit_ops |= progress;
    } while (progress);
 
-   if (lowered_64bit_ops) {
-      OPT(nir_lower_constant_initializers, nir_var_function_temp);
-      OPT(nir_lower_returns);
-      OPT(nir_inline_functions);
-      OPT(nir_opt_deref);
-   }
-
-   const nir_function *entry_point = nir_shader_get_entrypoint(nir)->function;
-   foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
-      if (func != entry_point) {
-         exec_node_remove(&func->node);
-      }
-   }
-   assert(exec_list_length(&nir->functions) == 1);
-
    if (nir->info.stage == MESA_SHADER_GEOMETRY)
       OPT(nir_lower_gs_intrinsics);
 
@@ -1098,7 +1084,7 @@ brw_nir_create_passthrough_tcs(void *mem_ctx, const struct brw_compiler *compile
 
    nir_validate_shader(nir, "in brw_nir_create_passthrough_tcs");
 
-   nir = brw_preprocess_nir(compiler, nir);
+   nir = brw_preprocess_nir(compiler, nir, NULL);
 
    return nir;
 }

src/intel/compiler/brw_nir.h

@@ -93,7 +93,8 @@ enum {
 void brw_nir_analyze_boolean_resolves(nir_shader *nir);
 
 nir_shader *brw_preprocess_nir(const struct brw_compiler *compiler,
-                               nir_shader *nir);
+                               nir_shader *nir,
+                               const nir_shader *softfp64);
 
 void
 brw_nir_link_shaders(const struct brw_compiler *compiler,

src/intel/vulkan/anv_pipeline.c

@@ -231,7 +231,7 @@ anv_shader_compile_to_nir(struct anv_device *device,
    /* Vulkan uses the separate-shader linking model */
    nir->info.separate_shader = true;
 
-   nir = brw_preprocess_nir(compiler, nir);
+   nir = brw_preprocess_nir(compiler, nir, NULL);
 
    return nir;
 }

src/mesa/drivers/dri/i965/brw_program.c

@@ -112,14 +112,14 @@ brw_create_nir(struct brw_context *brw,
 
    nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
 
+   nir_shader *softfp64 = NULL;
    if ((options->lower_doubles_options & nir_lower_fp64_full_software) &&
        nir->info.uses_64bit) {
-      nir_shader *fp64 = glsl_float64_funcs_to_nir(ctx, options);
+      softfp64 = glsl_float64_funcs_to_nir(ctx, options);
-      ralloc_steal(ralloc_parent(nir), fp64);
+      ralloc_steal(ralloc_parent(nir), softfp64);
-      exec_list_append(&nir->functions, &fp64->functions);
    }
 
-   nir = brw_preprocess_nir(brw->screen->compiler, nir);
+   nir = brw_preprocess_nir(brw->screen->compiler, nir, softfp64);
 
    NIR_PASS_V(nir, brw_nir_lower_image_load_store, devinfo);
 

src/mesa/state_tracker/st_glsl_to_nir.cpp

@@ -379,11 +379,11 @@ st_glsl_to_nir(struct st_context *st, struct gl_program *prog,
    }
 
    nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
+   nir_shader *softfp64 = NULL;
    if (nir->info.uses_64bit &&
        (options->lower_doubles_options & nir_lower_fp64_full_software) != 0) {
-      nir_shader *fp64 = glsl_float64_funcs_to_nir(st->ctx, options);
+      softfp64 = glsl_float64_funcs_to_nir(st->ctx, options);
-      ralloc_steal(ralloc_parent(nir), fp64);
+      ralloc_steal(ralloc_parent(nir), softfp64);
-      exec_list_append(&nir->functions, &fp64->functions);
    }
 
    nir_variable_mode mask =
@@ -424,26 +424,11 @@ st_glsl_to_nir(struct st_context *st, struct gl_program *prog,
          }
          if (options->lower_doubles_options) {
             NIR_PASS(progress, nir, nir_lower_doubles,
-                     options->lower_doubles_options);
+                     softfp64, options->lower_doubles_options);
          }
          NIR_PASS(progress, nir, nir_opt_algebraic);
          lowered_64bit_ops |= progress;
       } while (progress);
-
-      if (lowered_64bit_ops) {
-         NIR_PASS_V(nir, nir_lower_constant_initializers, nir_var_function_temp);
-         NIR_PASS_V(nir, nir_lower_returns);
-         NIR_PASS_V(nir, nir_inline_functions);
-         NIR_PASS_V(nir, nir_opt_deref);
-      }
-
-      const nir_function *entry_point = nir_shader_get_entrypoint(nir)->function;
-      foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
-         if (func != entry_point) {
-            exec_node_remove(&func->node);
-         }
-      }
-      assert(exec_list_length(&nir->functions) == 1);
    }
 
    st_nir_opts(nir, is_scalar);