OpenHarmony/third_party_mesa3d
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
55f8a01acae7c4171ccfef301e48dc3cc67f5836
third_party_mesa3d/src/broadcom/compiler/qpu_schedule.c

1391 lines
49 KiB
C
Raw Normal View History

broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
/*
* Copyright © 2010 Intel Corporation
* Copyright © 2014-2017 Broadcom
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
/**
* @file
*
* The basic model of the list scheduler is to take a basic block, compute a
* DAG of the dependencies, and make a list of the DAG heads. Heuristically
* pick a DAG head, then put all the children that are now DAG heads into the
* list of things to schedule.
*
* The goal of scheduling here is to pack pairs of operations together in a
* single QPU instruction.
*/
#include "qpu/qpu_disasm.h"
#include "v3d_compiler.h"
#include "util/ralloc.h"
static bool debug;
struct schedule_node_child;
struct schedule_node {
struct list_head link;
struct qinst *inst;
struct schedule_node_child *children;
uint32_t child_count;
uint32_t child_array_size;
uint32_t parent_count;
/* Longest cycles + instruction_latency() of any parent of this node. */
uint32_t unblocked_time;
/**
* Minimum number of cycles from scheduling this instruction until the
* end of the program, based on the slowest dependency chain through
* the children.
*/
uint32_t delay;
/**
* cycles between this instruction being scheduled and when its result
* can be consumed.
*/
uint32_t latency;
};
struct schedule_node_child {
struct schedule_node *node;
bool write_after_read;
};
/* When walking the instructions in reverse, we need to swap before/after in
* add_dep().
*/
enum direction { F, R };
struct schedule_state {
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
const struct v3d_device_info *devinfo;
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
struct schedule_node *last_r[6];
struct schedule_node *last_rf[64];
struct schedule_node *last_sf;
struct schedule_node *last_vpm_read;
struct schedule_node *last_tmu_write;
struct schedule_node *last_tlb;
struct schedule_node *last_vpm;
struct schedule_node *last_unif;
struct schedule_node *last_rtop;
enum direction dir;
/* Estimated cycle when the current instruction would start. */
uint32_t time;
};
static void
add_dep(struct schedule_state *state,
struct schedule_node *before,
struct schedule_node *after,
bool write)
{
bool write_after_read = !write && state->dir == R;
if (!before || !after)
return;
assert(before != after);
if (state->dir == R) {
struct schedule_node *t = before;
before = after;
after = t;
}
for (int i = 0; i < before->child_count; i++) {
if (before->children[i].node == after &&
(before->children[i].write_after_read == write_after_read)) {
return;
}
}
if (before->child_array_size <= before->child_count) {
before->child_array_size = MAX2(before->child_array_size * 2, 16);
before->children = reralloc(before, before->children,
struct schedule_node_child,
before->child_array_size);
}
before->children[before->child_count].node = after;
before->children[before->child_count].write_after_read =
write_after_read;
before->child_count++;
after->parent_count++;
}
static void
add_read_dep(struct schedule_state *state,
struct schedule_node *before,
struct schedule_node *after)
{
add_dep(state, before, after, false);
}
static void
add_write_dep(struct schedule_state *state,
struct schedule_node **before,
struct schedule_node *after)
{
add_dep(state, *before, after, true);
*before = after;
}
static bool
qpu_inst_is_tlb(const struct v3d_qpu_instr *inst)
{
if (inst->type != V3D_QPU_INSTR_TYPE_ALU)
return false;
if (inst->alu.add.magic_write &&
(inst->alu.add.waddr == V3D_QPU_WADDR_TLB ||
inst->alu.add.waddr == V3D_QPU_WADDR_TLBU))
return true;
if (inst->alu.mul.magic_write &&
(inst->alu.mul.waddr == V3D_QPU_WADDR_TLB ||
inst->alu.mul.waddr == V3D_QPU_WADDR_TLBU))
return true;
return false;
}
static void
process_mux_deps(struct schedule_state *state, struct schedule_node *n,
enum v3d_qpu_mux mux)
{
switch (mux) {
case V3D_QPU_MUX_A:
add_read_dep(state, state->last_rf[n->inst->qpu.raddr_a], n);
break;
case V3D_QPU_MUX_B:
add_read_dep(state, state->last_rf[n->inst->qpu.raddr_b], n);
break;
default:
add_read_dep(state, state->last_r[mux - V3D_QPU_MUX_R0], n);
break;
}
}
static void
process_waddr_deps(struct schedule_state *state, struct schedule_node *n,
uint32_t waddr, bool magic)
{
if (!magic) {
add_write_dep(state, &state->last_rf[waddr], n);
} else if (v3d_qpu_magic_waddr_is_tmu(waddr)) {
add_write_dep(state, &state->last_tmu_write, n);
} else if (v3d_qpu_magic_waddr_is_sfu(waddr)) {
/* Handled by v3d_qpu_writes_r4() check. */
} else {
switch (waddr) {
case V3D_QPU_WADDR_R0:
case V3D_QPU_WADDR_R1:
case V3D_QPU_WADDR_R2:
add_write_dep(state,
&state->last_r[waddr - V3D_QPU_WADDR_R0],
n);
break;
broadcom/vc5: Fix scheduling for a non-SFU R4 write after a dead R4 write. The v3d_qpu_writes_r*() were only checking for fixed-function accumulator writes, not normal ALU writes to those regs. Fixes fs-discard-exit-2 on simulation (but not HW).
2017-11-07 09:51:56 -08:00
case V3D_QPU_WADDR_R3:
case V3D_QPU_WADDR_R4:
case V3D_QPU_WADDR_R5:
/* Handled by v3d_qpu_writes_r*() checks below. */
break;
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
case V3D_QPU_WADDR_VPM:
case V3D_QPU_WADDR_VPMU:
add_write_dep(state, &state->last_vpm, n);
break;
case V3D_QPU_WADDR_TLB:
case V3D_QPU_WADDR_TLBU:
add_write_dep(state, &state->last_tlb, n);
break;
case V3D_QPU_WADDR_NOP:
break;
default:
fprintf(stderr, "Unknown waddr %d\n", waddr);
abort();
}
}
}
static void
process_cond_deps(struct schedule_state *state, struct schedule_node *n,
enum v3d_qpu_cond cond)
{
if (cond != V3D_QPU_COND_NONE)
add_read_dep(state, state->last_sf, n);
}
static void
process_pf_deps(struct schedule_state *state, struct schedule_node *n,
enum v3d_qpu_pf pf)
{
if (pf != V3D_QPU_PF_NONE)
add_write_dep(state, &state->last_sf, n);
}
static void
process_uf_deps(struct schedule_state *state, struct schedule_node *n,
enum v3d_qpu_uf uf)
{
if (uf != V3D_QPU_UF_NONE)
add_write_dep(state, &state->last_sf, n);
}
/**
* Common code for dependencies that need to be tracked both forward and
* backward.
*
* This is for things like "all reads of r4 have to happen between the r4
* writes that surround them".
*/
static void
calculate_deps(struct schedule_state *state, struct schedule_node *n)
{
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
const struct v3d_device_info *devinfo = state->devinfo;
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
struct qinst *qinst = n->inst;
struct v3d_qpu_instr *inst = &qinst->qpu;
if (inst->type == V3D_QPU_INSTR_TYPE_BRANCH) {
if (inst->branch.cond != V3D_QPU_BRANCH_COND_ALWAYS)
add_read_dep(state, state->last_sf, n);
/* XXX: BDI */
/* XXX: BDU */
/* XXX: ub */
/* XXX: raddr_a */
add_write_dep(state, &state->last_unif, n);
return;
}
assert(inst->type == V3D_QPU_INSTR_TYPE_ALU);
/* XXX: LOAD_IMM */
if (v3d_qpu_add_op_num_src(inst->alu.add.op) > 0)
process_mux_deps(state, n, inst->alu.add.a);
if (v3d_qpu_add_op_num_src(inst->alu.add.op) > 1)
process_mux_deps(state, n, inst->alu.add.b);
if (v3d_qpu_mul_op_num_src(inst->alu.mul.op) > 0)
process_mux_deps(state, n, inst->alu.mul.a);
if (v3d_qpu_mul_op_num_src(inst->alu.mul.op) > 1)
process_mux_deps(state, n, inst->alu.mul.b);
switch (inst->alu.add.op) {
case V3D_QPU_A_VPMSETUP:
/* Could distinguish read/write by unpacking the uniform. */
add_write_dep(state, &state->last_vpm, n);
add_write_dep(state, &state->last_vpm_read, n);
break;
case V3D_QPU_A_STVPMV:
case V3D_QPU_A_STVPMD:
case V3D_QPU_A_STVPMP:
add_write_dep(state, &state->last_vpm, n);
break;
case V3D_QPU_A_MSF:
add_read_dep(state, state->last_tlb, n);
break;
case V3D_QPU_A_SETMSF:
case V3D_QPU_A_SETREVF:
add_write_dep(state, &state->last_tlb, n);
break;
case V3D_QPU_A_FLAPUSH:
case V3D_QPU_A_FLBPUSH:
case V3D_QPU_A_VFLA:
case V3D_QPU_A_VFLNA:
case V3D_QPU_A_VFLB:
case V3D_QPU_A_VFLNB:
add_read_dep(state, state->last_sf, n);
break;
case V3D_QPU_A_FLBPOP:
add_write_dep(state, &state->last_sf, n);
break;
default:
break;
}
switch (inst->alu.mul.op) {
case V3D_QPU_M_MULTOP:
case V3D_QPU_M_UMUL24:
/* MULTOP sets rtop, and UMUL24 implicitly reads rtop and
* resets it to 0. We could possibly reorder umul24s relative
* to each other, but for now just keep all the MUL parts in
* order.
*/
add_write_dep(state, &state->last_rtop, n);
break;
default:
break;
}
if (inst->alu.add.op != V3D_QPU_A_NOP) {
process_waddr_deps(state, n, inst->alu.add.waddr,
inst->alu.add.magic_write);
}
if (inst->alu.mul.op != V3D_QPU_M_NOP) {
process_waddr_deps(state, n, inst->alu.mul.waddr,
inst->alu.mul.magic_write);
}
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
if (v3d_qpu_sig_writes_address(devinfo, &inst->sig)) {
process_waddr_deps(state, n, inst->sig_addr,
inst->sig_magic);
}
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
if (v3d_qpu_writes_r3(devinfo, inst))
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
add_write_dep(state, &state->last_r[3], n);
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
if (v3d_qpu_writes_r4(devinfo, inst))
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
add_write_dep(state, &state->last_r[4], n);
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
if (v3d_qpu_writes_r5(devinfo, inst))
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
add_write_dep(state, &state->last_r[5], n);
if (inst->sig.thrsw) {
/* All accumulator contents and flags are undefined after the
* switch.
*/
for (int i = 0; i < ARRAY_SIZE(state->last_r); i++)
add_write_dep(state, &state->last_r[i], n);
add_write_dep(state, &state->last_sf, n);
/* Scoreboard-locking operations have to stay after the last
* thread switch.
*/
add_write_dep(state, &state->last_tlb, n);
add_write_dep(state, &state->last_tmu_write, n);
}
if (inst->sig.ldtmu) {
/* TMU loads are coming from a FIFO, so ordering is important.
*/
add_write_dep(state, &state->last_tmu_write, n);
}
if (inst->sig.ldtlb | inst->sig.ldtlbu)
add_read_dep(state, state->last_tlb, n);
if (inst->sig.ldvpm)
add_write_dep(state, &state->last_vpm_read, n);
/* inst->sig.ldunif or sideband uniform read */
if (qinst->uniform != ~0)
add_write_dep(state, &state->last_unif, n);
process_cond_deps(state, n, inst->flags.ac);
process_cond_deps(state, n, inst->flags.mc);
process_pf_deps(state, n, inst->flags.apf);
process_pf_deps(state, n, inst->flags.mpf);
process_uf_deps(state, n, inst->flags.auf);
process_uf_deps(state, n, inst->flags.muf);
}
static void
calculate_forward_deps(struct v3d_compile *c, struct list_head *schedule_list)
{
struct schedule_state state;
memset(&state, 0, sizeof(state));
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
state.devinfo = c->devinfo;
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
state.dir = F;
list_for_each_entry(struct schedule_node, node, schedule_list, link)
calculate_deps(&state, node);
}
static void
calculate_reverse_deps(struct v3d_compile *c, struct list_head *schedule_list)
{
struct list_head *node;
struct schedule_state state;
memset(&state, 0, sizeof(state));
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
state.devinfo = c->devinfo;
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
state.dir = R;
for (node = schedule_list->prev; schedule_list != node; node = node->prev) {
calculate_deps(&state, (struct schedule_node *)node);
}
}
struct choose_scoreboard {
int tick;
int last_sfu_write_tick;
int last_ldvary_tick;
int last_uniforms_reset_tick;
uint32_t last_waddr_add, last_waddr_mul;
bool tlb_locked;
};
static bool
mux_reads_too_soon(struct choose_scoreboard *scoreboard,
const struct v3d_qpu_instr *inst, enum v3d_qpu_mux mux)
{
switch (mux) {
case V3D_QPU_MUX_A:
if (scoreboard->last_waddr_add == inst->raddr_a ||
scoreboard->last_waddr_mul == inst->raddr_a) {
return true;
}
break;
case V3D_QPU_MUX_B:
if (scoreboard->last_waddr_add == inst->raddr_b ||
scoreboard->last_waddr_mul == inst->raddr_b) {
return true;
}
break;
case V3D_QPU_MUX_R4:
if (scoreboard->tick - scoreboard->last_sfu_write_tick <= 2)
return true;
break;
case V3D_QPU_MUX_R5:
if (scoreboard->tick - scoreboard->last_ldvary_tick <= 1)
return true;
break;
default:
break;
}
return false;
}
static bool
reads_too_soon_after_write(struct choose_scoreboard *scoreboard,
struct qinst *qinst)
{
const struct v3d_qpu_instr *inst = &qinst->qpu;
/* XXX: Branching off of raddr. */
if (inst->type == V3D_QPU_INSTR_TYPE_BRANCH)
return false;
assert(inst->type == V3D_QPU_INSTR_TYPE_ALU);
if (inst->alu.add.op != V3D_QPU_A_NOP) {
if (v3d_qpu_add_op_num_src(inst->alu.add.op) > 0 &&
mux_reads_too_soon(scoreboard, inst, inst->alu.add.a)) {
return true;
}
if (v3d_qpu_add_op_num_src(inst->alu.add.op) > 1 &&
mux_reads_too_soon(scoreboard, inst, inst->alu.add.b)) {
return true;
}
}
if (inst->alu.mul.op != V3D_QPU_M_NOP) {
if (v3d_qpu_mul_op_num_src(inst->alu.mul.op) > 0 &&
mux_reads_too_soon(scoreboard, inst, inst->alu.mul.a)) {
return true;
}
if (v3d_qpu_mul_op_num_src(inst->alu.mul.op) > 1 &&
mux_reads_too_soon(scoreboard, inst, inst->alu.mul.b)) {
return true;
}
}
/* XXX: imm */
return false;
}
static bool
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
writes_too_soon_after_write(const struct v3d_device_info *devinfo,
struct choose_scoreboard *scoreboard,
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
struct qinst *qinst)
{
const struct v3d_qpu_instr *inst = &qinst->qpu;
/* Don't schedule any other r4 write too soon after an SFU write.
* This would normally be prevented by dependency tracking, but might
* occur if a dead SFU computation makes it to scheduling.
*/
if (scoreboard->tick - scoreboard->last_sfu_write_tick < 2 &&
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
v3d_qpu_writes_r4(devinfo, inst))
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
return true;
return false;
}
static bool
pixel_scoreboard_too_soon(struct choose_scoreboard *scoreboard,
const struct v3d_qpu_instr *inst)
{
return (scoreboard->tick == 0 && qpu_inst_is_tlb(inst));
}
static int
get_instruction_priority(const struct v3d_qpu_instr *inst)
{
uint32_t baseline_score;
uint32_t next_score = 0;
/* Schedule TLB operations as late as possible, to get more
* parallelism between shaders.
*/
if (qpu_inst_is_tlb(inst))
return next_score;
next_score++;
/* Schedule texture read results collection late to hide latency. */
if (inst->sig.ldtmu)
return next_score;
next_score++;
/* Default score for things that aren't otherwise special. */
baseline_score = next_score;
next_score++;
/* Schedule texture read setup early to hide their latency better. */
if (inst->type == V3D_QPU_INSTR_TYPE_ALU &&
((inst->alu.add.magic_write &&
v3d_qpu_magic_waddr_is_tmu(inst->alu.add.waddr)) ||
(inst->alu.mul.magic_write &&
v3d_qpu_magic_waddr_is_tmu(inst->alu.mul.waddr)))) {
return next_score;
}
next_score++;
return baseline_score;
}
static bool
qpu_magic_waddr_is_periph(enum v3d_qpu_waddr waddr)
{
return (v3d_qpu_magic_waddr_is_tmu(waddr) ||
v3d_qpu_magic_waddr_is_sfu(waddr) ||
v3d_qpu_magic_waddr_is_tlb(waddr) ||
v3d_qpu_magic_waddr_is_vpm(waddr) ||
v3d_qpu_magic_waddr_is_tsy(waddr));
}
static bool
qpu_accesses_peripheral(const struct v3d_qpu_instr *inst)
{
if (inst->type == V3D_QPU_INSTR_TYPE_ALU) {
if (inst->alu.add.op != V3D_QPU_A_NOP &&
inst->alu.add.magic_write &&
qpu_magic_waddr_is_periph(inst->alu.add.waddr)) {
return true;
}
broadcom/vc5: Don't pair VPMSETUP with other peripheral access. The specs don't say you can't, but pairing it with an SFU write on the 7268 breaks all our simple shader tests using gl_MVP * gl_Vertex.
2017-10-11 17:40:35 -07:00
if (inst->alu.add.op == V3D_QPU_A_VPMSETUP)
return true;
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
if (inst->alu.mul.op != V3D_QPU_M_NOP &&
inst->alu.mul.magic_write &&
qpu_magic_waddr_is_periph(inst->alu.mul.waddr)) {
return true;
}
}
return (inst->sig.ldvpm ||
inst->sig.ldtmu ||
inst->sig.ldtlb ||
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
inst->sig.ldtlbu ||
inst->sig.wrtmuc);
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
}
static bool
qpu_merge_inst(const struct v3d_device_info *devinfo,
struct v3d_qpu_instr *result,
const struct v3d_qpu_instr *a,
const struct v3d_qpu_instr *b)
{
if (a->type != V3D_QPU_INSTR_TYPE_ALU ||
b->type != V3D_QPU_INSTR_TYPE_ALU) {
return false;
}
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
/* Can't do more than one peripheral access in an instruction.
*
* XXX: V3D 4.1 allows TMU read along with a VPM read or write, and
* WRTMUC with a TMU magic register write (other than tmuc).
*/
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
if (qpu_accesses_peripheral(a) && qpu_accesses_peripheral(b))
return false;
struct v3d_qpu_instr merge = *a;
if (b->alu.add.op != V3D_QPU_A_NOP) {
if (a->alu.add.op != V3D_QPU_A_NOP)
return false;
merge.alu.add = b->alu.add;
merge.flags.ac = b->flags.ac;
merge.flags.apf = b->flags.apf;
merge.flags.auf = b->flags.auf;
}
if (b->alu.mul.op != V3D_QPU_M_NOP) {
if (a->alu.mul.op != V3D_QPU_M_NOP)
return false;
merge.alu.mul = b->alu.mul;
merge.flags.mc = b->flags.mc;
merge.flags.mpf = b->flags.mpf;
merge.flags.muf = b->flags.muf;
}
if (v3d_qpu_uses_mux(b, V3D_QPU_MUX_A)) {
if (v3d_qpu_uses_mux(a, V3D_QPU_MUX_A) &&
a->raddr_a != b->raddr_a) {
return false;
}
merge.raddr_a = b->raddr_a;
}
if (v3d_qpu_uses_mux(b, V3D_QPU_MUX_B)) {
if (v3d_qpu_uses_mux(a, V3D_QPU_MUX_B) &&
a->raddr_b != b->raddr_b) {
return false;
}
merge.raddr_b = b->raddr_b;
}
merge.sig.thrsw |= b->sig.thrsw;
merge.sig.ldunif |= b->sig.ldunif;
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
merge.sig.ldunifrf |= b->sig.ldunifrf;
merge.sig.ldunifa |= b->sig.ldunifa;
merge.sig.ldunifarf |= b->sig.ldunifarf;
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
merge.sig.ldtmu |= b->sig.ldtmu;
merge.sig.ldvary |= b->sig.ldvary;
merge.sig.ldvpm |= b->sig.ldvpm;
merge.sig.small_imm |= b->sig.small_imm;
merge.sig.ldtlb |= b->sig.ldtlb;
merge.sig.ldtlbu |= b->sig.ldtlbu;
merge.sig.ucb |= b->sig.ucb;
merge.sig.rotate |= b->sig.rotate;
merge.sig.wrtmuc |= b->sig.wrtmuc;
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
if (v3d_qpu_sig_writes_address(devinfo, &a->sig) &&
v3d_qpu_sig_writes_address(devinfo, &b->sig))
return false;
merge.sig_addr |= b->sig_addr;
merge.sig_magic |= b->sig_magic;
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
uint64_t packed;
bool ok = v3d_qpu_instr_pack(devinfo, &merge, &packed);
*result = merge;
/* No modifying the real instructions on failure. */
assert(ok || (a != result && b != result));
return ok;
}
static struct schedule_node *
choose_instruction_to_schedule(const struct v3d_device_info *devinfo,
struct choose_scoreboard *scoreboard,
struct list_head *schedule_list,
struct schedule_node *prev_inst)
{
struct schedule_node *chosen = NULL;
int chosen_prio = 0;
/* Don't pair up anything with a thread switch signal -- emit_thrsw()
* will handle pairing it along with filling the delay slots.
*/
if (prev_inst) {
if (prev_inst->inst->qpu.sig.thrsw)
return NULL;
}
list_for_each_entry(struct schedule_node, n, schedule_list, link) {
const struct v3d_qpu_instr *inst = &n->inst->qpu;
/* Don't choose the branch instruction until it's the last one
* left. We'll move it up to fit its delay slots after we
* choose it.
*/
if (inst->type == V3D_QPU_INSTR_TYPE_BRANCH &&
!list_is_singular(schedule_list)) {
continue;
}
/* "An instruction must not read from a location in physical
* regfile A or B that was written to by the previous
* instruction."
*/
if (reads_too_soon_after_write(scoreboard, n->inst))
continue;
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
if (writes_too_soon_after_write(devinfo, scoreboard, n->inst))
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
continue;
/* "A scoreboard wait must not occur in the first two
* instructions of a fragment shader. This is either the
* explicit Wait for Scoreboard signal or an implicit wait
* with the first tile-buffer read or write instruction."
*/
if (pixel_scoreboard_too_soon(scoreboard, inst))
continue;
/* ldunif and ldvary both write r5, but ldunif does so a tick
* sooner. If the ldvary's r5 wasn't used, then ldunif might
* otherwise get scheduled so ldunif and ldvary try to update
* r5 in the same tick.
*/
broadcom/vc5: Add support for V3Dv4 signal bits. The WRTMUC replaces the implicit uniform loads in the first two texture instructions. LDVPM disappears in favor of an ALU op. LDVARY, LDTMU, LDTLB, and LDUNIF*RF now write to arbitrary registers, which required passing the devinfo through to a few more functions.
2018-01-03 21:42:33 -08:00
if ((inst->sig.ldunif || inst->sig.ldunifa) &&
broadcom: Add VC5 NIR compiler. This is a pretty straightforward fork of VC4's NIR compiler to VC5. The condition codes, registers, and I/O have all changed, making the backend hard to share, though their heritage is still recognizable. v2: Move to src/broadcom/compiler to match intel's layout, rename more "vc5" to "v3d", rename QIR to VIR ("V3D IR") to avoid symbol conflicts with vc4, use new v3d_debug header, add compiler init/free functions, do texture swizzling in NIR to allow optimization.
2017-02-03 10:24:14 -08:00
scoreboard->tick == scoreboard->last_ldvary_tick + 1) {
continue;
}
/* If we're trying to pair with another instruction, check
* that they're compatible.
*/
if (prev_inst) {
/* Don't pair up a thread switch signal -- we'll
* handle pairing it when we pick it on its own.
*/
if (inst->sig.thrsw)
continue;
if (prev_inst->inst->uniform != -1 &&
n->inst->uniform != -1)
continue;
/* Don't merge in something that will lock the TLB.
* Hopwefully what we have in inst will release some
* other instructions, allowing us to delay the
* TLB-locking instruction until later.
*/
if (!scoreboard->tlb_locked && qpu_inst_is_tlb(inst))
continue;
struct v3d_qpu_instr merged_inst;
if (!qpu_merge_inst(devinfo, &merged_inst,
&prev_inst->inst->qpu, inst)) {
continue;
}
}
int prio = get_instruction_priority(inst);
/* Found a valid instruction. If nothing better comes along,
* this one works.
*/
if (!chosen) {
chosen = n;
chosen_prio = prio;
continue;
}
if (prio > chosen_prio) {
chosen = n;
chosen_prio = prio;
} else if (prio < chosen_prio) {
continue;
}
if (n->delay > chosen->delay) {
chosen = n;
chosen_prio = prio;
} else if (n->delay < chosen->delay) {
continue;
}
}
return chosen;
}
static void
update_scoreboard_for_magic_waddr(struct choose_scoreboard *scoreboard,
enum v3d_qpu_waddr waddr)
{
if (v3d_qpu_magic_waddr_is_sfu(waddr))
scoreboard->last_sfu_write_tick = scoreboard->tick;
}
static void
update_scoreboard_for_chosen(struct choose_scoreboard *scoreboard,
const struct v3d_qpu_instr *inst)
{
scoreboard->last_waddr_add = ~0;
scoreboard->last_waddr_mul = ~0;
if (inst->type == V3D_QPU_INSTR_TYPE_BRANCH)
return;
assert(inst->type == V3D_QPU_INSTR_TYPE_ALU);
if (inst->alu.add.op != V3D_QPU_A_NOP) {
if (inst->alu.add.magic_write) {
update_scoreboard_for_magic_waddr(scoreboard,
inst->alu.add.waddr);
} else {
scoreboard->last_waddr_add = inst->alu.add.waddr;
}
}
if (inst->alu.mul.op != V3D_QPU_M_NOP) {
if (inst->alu.mul.magic_write) {
update_scoreboard_for_magic_waddr(scoreboard,
inst->alu.mul.waddr);
} else {
scoreboard->last_waddr_mul = inst->alu.mul.waddr;
}
}
if (inst->sig.ldvary)
scoreboard->last_ldvary_tick = scoreboard->tick;
if (qpu_inst_is_tlb(inst))
scoreboard->tlb_locked = true;
}
static void
dump_state(const struct v3d_device_info *devinfo,
struct list_head *schedule_list)
{
list_for_each_entry(struct schedule_node, n, schedule_list, link) {
fprintf(stderr, " t=%4d: ", n->unblocked_time);
v3d_qpu_dump(devinfo, &n->inst->qpu);
fprintf(stderr, "\n");
for (int i = 0; i < n->child_count; i++) {
struct schedule_node *child = n->children[i].node;
if (!child)
continue;
fprintf(stderr, " - ");
v3d_qpu_dump(devinfo, &child->inst->qpu);
fprintf(stderr, " (%d parents, %c)\n",
child->parent_count,
n->children[i].write_after_read ? 'w' : 'r');
}
}
}
static uint32_t magic_waddr_latency(enum v3d_qpu_waddr waddr,
const struct v3d_qpu_instr *after)
{
/* Apply some huge latency between texture fetch requests and getting
* their results back.
*
* FIXME: This is actually pretty bogus. If we do:
*
* mov tmu0_s, a
* <a bit of math>
* mov tmu0_s, b
* load_tmu0
* <more math>
* load_tmu0
*
* we count that as worse than
*
* mov tmu0_s, a
* mov tmu0_s, b
* <lots of math>
* load_tmu0
* <more math>
* load_tmu0
*
* because we associate the first load_tmu0 with the *second* tmu0_s.
*/
if (v3d_qpu_magic_waddr_is_tmu(waddr) && after->sig.ldtmu)
return 100;
/* Assume that anything depending on us is consuming the SFU result. */
if (v3d_qpu_magic_waddr_is_sfu(waddr))
return 3;
return 1;
}
static uint32_t
instruction_latency(struct schedule_node *before, struct schedule_node *after)
{
const struct v3d_qpu_instr *before_inst = &before->inst->qpu;
const struct v3d_qpu_instr *after_inst = &after->inst->qpu;
uint32_t latency = 1;
if (before_inst->type != V3D_QPU_INSTR_TYPE_ALU ||
after_inst->type != V3D_QPU_INSTR_TYPE_ALU)
return latency;
if (before_inst->alu.add.magic_write) {
latency = MAX2(latency,
magic_waddr_latency(before_inst->alu.add.waddr,
after_inst));
}
if (before_inst->alu.mul.magic_write) {
latency = MAX2(latency,
magic_waddr_latency(before_inst->alu.mul.waddr,
after_inst));
}
return latency;
}
/** Recursive computation of the delay member of a node. */
static void
compute_delay(struct schedule_node *n)
{
if (!n->child_count) {
n->delay = 1;
} else {
for (int i = 0; i < n->child_count; i++) {
if (!n->children[i].node->delay)
compute_delay(n->children[i].node);
n->delay = MAX2(n->delay,
n->children[i].node->delay +
instruction_latency(n, n->children[i].node));
}
}
}
static void
mark_instruction_scheduled(struct list_head *schedule_list,
uint32_t time,
struct schedule_node *node,
bool war_only)
{
if (!node)
return;
for (int i = node->child_count - 1; i >= 0; i--) {
struct schedule_node *child =
node->children[i].node;
if (!child)
continue;
if (war_only && !node->children[i].write_after_read)
continue;
/* If the requirement is only that the node not appear before
* the last read of its destination, then it can be scheduled
* immediately after (or paired with!) the thing reading the
* destination.
*/
uint32_t latency = 0;
if (!war_only) {
latency = instruction_latency(node,
node->children[i].node);
}
child->unblocked_time = MAX2(child->unblocked_time,
time + latency);
child->parent_count--;
if (child->parent_count == 0)
list_add(&child->link, schedule_list);
node->children[i].node = NULL;
}
}
static struct qinst *
vir_nop()
{
struct qreg undef = { QFILE_NULL, 0 };
struct qinst *qinst = vir_add_inst(V3D_QPU_A_NOP, undef, undef, undef);
return qinst;
}
#if 0
static struct qinst *
nop_after(struct qinst *inst)
{
struct qinst *q = vir_nop();
list_add(&q->link, &inst->link);
return q;
}
/**
* Emits a THRSW/LTHRSW signal in the stream, trying to move it up to pair
* with another instruction.
*/
static void
emit_thrsw(struct v3d_compile *c,
struct choose_scoreboard *scoreboard,
const struct v3d_qpu_instr *inst)
{
/* There should be nothing in a thrsw inst being scheduled other than
* the signal bits.
*/
assert(inst->type == V3D_QPU_INSTR_TYPE_ALU);
assert(inst->alu.add.op == V3D_QPU_A_NOP);
assert(inst->alu.mul.op == V3D_QPU_M_NOP);
/* Try to find an earlier scheduled instruction that we can merge the
* thrsw into.
*/
int thrsw_ip = c->qpu_inst_count;
for (int i = 1; i <= MIN2(c->qpu_inst_count, 3); i++) {
uint64_t prev_instr = c->qpu_insts[c->qpu_inst_count - i];
uint32_t prev_sig = QPU_GET_FIELD(prev_instr, QPU_SIG);
if (prev_sig == QPU_SIG_NONE)
thrsw_ip = c->qpu_inst_count - i;
}
if (thrsw_ip != c->qpu_inst_count) {
/* Merge the thrsw into the existing instruction. */
c->qpu_insts[thrsw_ip] =
QPU_UPDATE_FIELD(c->qpu_insts[thrsw_ip], sig, QPU_SIG);
} else {
qpu_serialize_one_inst(c, inst);
update_scoreboard_for_chosen(scoreboard, inst);
}
/* Fill the delay slots. */
while (c->qpu_inst_count < thrsw_ip + 3) {
update_scoreboard_for_chosen(scoreboard, v3d_qpu_nop());
qpu_serialize_one_inst(c, v3d_qpu_nop());
}
}
#endif
static uint32_t
schedule_instructions(struct v3d_compile *c,
struct choose_scoreboard *scoreboard,
struct qblock *block,
struct list_head *schedule_list,
enum quniform_contents *orig_uniform_contents,
uint32_t *orig_uniform_data,
uint32_t *next_uniform)
{
const struct v3d_device_info *devinfo = c->devinfo;
uint32_t time = 0;
if (debug) {
fprintf(stderr, "initial deps:\n");
dump_state(devinfo, schedule_list);
fprintf(stderr, "\n");
}
/* Remove non-DAG heads from the list. */
list_for_each_entry_safe(struct schedule_node, n, schedule_list, link) {
if (n->parent_count != 0)
list_del(&n->link);
}
while (!list_empty(schedule_list)) {
struct schedule_node *chosen =
choose_instruction_to_schedule(devinfo,
scoreboard,
schedule_list,
NULL);
struct schedule_node *merge = NULL;
/* If there are no valid instructions to schedule, drop a NOP
* in.
*/
struct qinst *qinst = chosen ? chosen->inst : vir_nop();
struct v3d_qpu_instr *inst = &qinst->qpu;
if (debug) {
fprintf(stderr, "t=%4d: current list:\n",
time);
dump_state(devinfo, schedule_list);
fprintf(stderr, "t=%4d: chose: ", time);
v3d_qpu_dump(devinfo, inst);
fprintf(stderr, "\n");
}
/* Schedule this instruction onto the QPU list. Also try to
* find an instruction to pair with it.
*/
if (chosen) {
time = MAX2(chosen->unblocked_time, time);
list_del(&chosen->link);
mark_instruction_scheduled(schedule_list, time,
chosen, true);
merge = choose_instruction_to_schedule(devinfo,
scoreboard,
schedule_list,
chosen);
if (merge) {
time = MAX2(merge->unblocked_time, time);
list_del(&merge->link);
(void)qpu_merge_inst(devinfo, inst,
inst, &merge->inst->qpu);
if (merge->inst->uniform != -1) {
chosen->inst->uniform =
merge->inst->uniform;
}
if (debug) {
fprintf(stderr, "t=%4d: merging: ",
time);
v3d_qpu_dump(devinfo, &merge->inst->qpu);
fprintf(stderr, "\n");
fprintf(stderr, " result: ");
v3d_qpu_dump(devinfo, inst);
fprintf(stderr, "\n");
}
}
}
/* Update the uniform index for the rewritten location --
* branch target updating will still need to change
* c->uniform_data[] using this index.
*/
if (qinst->uniform != -1) {
if (inst->type == V3D_QPU_INSTR_TYPE_BRANCH)
block->branch_uniform = *next_uniform;
c->uniform_data[*next_uniform] =
orig_uniform_data[qinst->uniform];
c->uniform_contents[*next_uniform] =
orig_uniform_contents[qinst->uniform];
qinst->uniform = *next_uniform;
(*next_uniform)++;
}
if (debug) {
fprintf(stderr, "\n");
}
/* Now that we've scheduled a new instruction, some of its
* children can be promoted to the list of instructions ready to
* be scheduled. Update the children's unblocked time for this
* DAG edge as we do so.
*/
mark_instruction_scheduled(schedule_list, time, chosen, false);
if (merge) {
mark_instruction_scheduled(schedule_list, time, merge,
false);
/* The merged VIR instruction doesn't get re-added to the
* block, so free it now.
*/
free(merge->inst);
}
if (0 && inst->sig.thrsw) {
/* XXX emit_thrsw(c, scoreboard, qinst); */
} else {
c->qpu_inst_count++;
list_addtail(&qinst->link, &block->instructions);
update_scoreboard_for_chosen(scoreboard, inst);
}
scoreboard->tick++;
time++;
if (inst->type == V3D_QPU_INSTR_TYPE_BRANCH ||
inst->sig.thrsw /* XXX */) {
block->branch_qpu_ip = c->qpu_inst_count - 1;
/* Fill the delay slots.
*
* We should fill these with actual instructions,
* instead, but that will probably need to be done
* after this, once we know what the leading
* instructions of the successors are (so we can
* handle A/B register file write latency)
*/
/* XXX: scoreboard */
int slots = (inst->type == V3D_QPU_INSTR_TYPE_BRANCH ?
3 : 2);
for (int i = 0; i < slots; i++) {
struct qinst *nop = vir_nop();
list_addtail(&nop->link, &block->instructions);
update_scoreboard_for_chosen(scoreboard,
&nop->qpu);
c->qpu_inst_count++;
scoreboard->tick++;
time++;
}
}
}
return time;
}
static uint32_t
qpu_schedule_instructions_block(struct v3d_compile *c,
struct choose_scoreboard *scoreboard,
struct qblock *block,
enum quniform_contents *orig_uniform_contents,
uint32_t *orig_uniform_data,
uint32_t *next_uniform)
{
void *mem_ctx = ralloc_context(NULL);
struct list_head schedule_list;
list_inithead(&schedule_list);
/* Wrap each instruction in a scheduler structure. */
while (!list_empty(&block->instructions)) {
struct qinst *qinst = (struct qinst *)block->instructions.next;
struct schedule_node *n =
rzalloc(mem_ctx, struct schedule_node);
n->inst = qinst;
list_del(&qinst->link);
list_addtail(&n->link, &schedule_list);
}
calculate_forward_deps(c, &schedule_list);
calculate_reverse_deps(c, &schedule_list);
list_for_each_entry(struct schedule_node, n, &schedule_list, link) {
compute_delay(n);
}
uint32_t cycles = schedule_instructions(c, scoreboard, block,
&schedule_list,
orig_uniform_contents,
orig_uniform_data,
next_uniform);
ralloc_free(mem_ctx);
return cycles;
}
static void
qpu_set_branch_targets(struct v3d_compile *c)
{
vir_for_each_block(block, c) {
/* The end block of the program has no branch. */
if (!block->successors[0])
continue;
/* If there was no branch instruction, then the successor
* block must follow immediately after this one.
*/
if (block->branch_qpu_ip == ~0) {
assert(block->end_qpu_ip + 1 ==
block->successors[0]->start_qpu_ip);
continue;
}
/* Walk back through the delay slots to find the branch
* instr.
*/
struct list_head *entry = block->instructions.prev;
for (int i = 0; i < 3; i++)
entry = entry->prev;
struct qinst *branch = container_of(entry, branch, link);
assert(branch->qpu.type == V3D_QPU_INSTR_TYPE_BRANCH);
/* Make sure that the if-we-don't-jump
* successor was scheduled just after the
* delay slots.
*/
assert(!block->successors[1] ||
block->successors[1]->start_qpu_ip ==
block->branch_qpu_ip + 4);
branch->qpu.branch.offset =
((block->successors[0]->start_qpu_ip -
(block->branch_qpu_ip + 4)) *
sizeof(uint64_t));
/* Set up the relative offset to jump in the
* uniform stream.
*
* Use a temporary here, because
* uniform_data[inst->uniform] may be shared
* between multiple instructions.
*/
assert(c->uniform_contents[branch->uniform] == QUNIFORM_CONSTANT);
c->uniform_data[branch->uniform] =
(block->successors[0]->start_uniform -
(block->branch_uniform + 1)) * 4;
}
}
uint32_t
v3d_qpu_schedule_instructions(struct v3d_compile *c)
{
const struct v3d_device_info *devinfo = c->devinfo;
/* We reorder the uniforms as we schedule instructions, so save the
* old data off and replace it.
*/
uint32_t *uniform_data = c->uniform_data;
enum quniform_contents *uniform_contents = c->uniform_contents;
c->uniform_contents = ralloc_array(c, enum quniform_contents,
c->num_uniforms);
c->uniform_data = ralloc_array(c, uint32_t, c->num_uniforms);
c->uniform_array_size = c->num_uniforms;
uint32_t next_uniform = 0;
struct choose_scoreboard scoreboard;
memset(&scoreboard, 0, sizeof(scoreboard));
scoreboard.last_waddr_add = ~0;
scoreboard.last_waddr_mul = ~0;
scoreboard.last_ldvary_tick = -10;
scoreboard.last_sfu_write_tick = -10;
scoreboard.last_uniforms_reset_tick = -10;
if (debug) {
fprintf(stderr, "Pre-schedule instructions\n");
vir_for_each_block(block, c) {
fprintf(stderr, "BLOCK %d\n", block->index);
list_for_each_entry(struct qinst, qinst,
&block->instructions, link) {
v3d_qpu_dump(devinfo, &qinst->qpu);
fprintf(stderr, "\n");
}
}
fprintf(stderr, "\n");
}
uint32_t cycles = 0;
vir_for_each_block(block, c) {
block->start_qpu_ip = c->qpu_inst_count;
block->branch_qpu_ip = ~0;
block->start_uniform = next_uniform;
cycles += qpu_schedule_instructions_block(c,
&scoreboard,
block,
uniform_contents,
uniform_data,
&next_uniform);
block->end_qpu_ip = c->qpu_inst_count - 1;
}
qpu_set_branch_targets(c);
assert(next_uniform == c->num_uniforms);
return cycles;
}
Reference in New Issue Copy Permalink