// PlayAgain's WebAssembly JIT for PPSSPP's IR, milestones M1 to M3 (see pa_wasmjit.h, and
// docs/superpowers/specs/2026-11-09-psp-wasm-jit-design.md for the why of each rule here).
//
// What a compiled block is: one function `(param $mips i32) (result i32)` in a module of up to 64,
// returning the PC to go on at. It runs exactly the IR IRInterpret would run for the block, including
// the interpreter-only Opt* ops, and copies the interpreter's C, not the PSP manual, so the two agree
// bit for bit on the same engine:
// - GPRs, lo, hi, fpcond, the IR temps and the VFPU control words read as integers get an i32 local each,
//   loaded on first use. M3: written back, not through: a write stays in its local, marked dirty, and every
//   dirty one is stored on each path out of the block and before each call to C that reads registers
//   (a syscall, Interpret, a replacement, an island); after those, every local counts as unloaded. The
//   downcount lives in a local too, stored before those calls and on the way out, read again after them.
//   An FPR lies over GPR-class register 32 + f: an FPR read stores that register first if it is dirty,
//   an FPR write drops its local.
// - M3: a register the block set to a constant (SetConst, and moves and the *Const ops of one) is known
//   while compiling: a guest access based on it is one constant address, an ExitToReg on it a constant exit.
// - FPR and VFPU values stay in memory, copied as bits (LoadFloat, StoreFloat, FMov, SetConstF and
//   the Vec4 loads and stores are plain copies).
// - M2, the FPU, Vec4 and the VFPU: each op is the wasm its IRInterpreter.cpp handler compiles to, operand
//   order and load order included (checked in IRInterpret's disassembly): f32 ops with the handler's
//   special cases as selects (FMul's inf * 0, FSqrt's negatives, vfpu_minmax_key, the clamps), the Vec4
//   ops on v128 as Emscripten's SSE headers compile them (Vec4Min/Max/Slt/Sge and CvtScaledSW as clang
//   vectorises their loops), the compares into fpcond or VFPU_CTRL_CC (kept in its local). The VFPU's own
//   functions (vfpu_dot, sin, rcp...) and the rounding conversions are calls to C wrappers that copy the
//   handlers. Div and DivU branch around wasm's trapping division. The one thing that can differ is what
//   wasm leaves to the engine: the NaN an add or a multiply of two different NaNs gives (README "NaNs").
// - Guest memory is base + ((rs + C) & MEMVIEW32_MASK), added before masking as the interpreter does;
//   base is one 256 MB block (ppsspp-3039c4e.patch), so nothing can fault.
// - The Downcounts of the block are folded: each path out charges the sum of the Downcounts on it,
//   once, right before it returns or calls out (exactly the interpreter's total at each point where
//   the downcount is read).
// - M3, links: a block goes on to the next one inside WebAssembly wherever IRInterpret's IR_EXIT would
//   chain (the downcount not negative, the target a valid address with a block in the shadow), and only
//   where that block is compiled: an exit to the block's own start is a wasm loop (with its registers
//   kept in locals across iterations); a constant exit to a block of the same module a direct tail call
//   (return_call) after checking the shadow still holds it and the epoch is the same; a constant exit
//   anywhere else a tail call through the target's link slot, a table slot per block shadow entry that
//   holds the stub (it returns the PC) until the block there is compiled and again once it is not; an
//   ExitToReg or ExitToPC looks the shadow up inline and calls the slot a CallWasm head names. Each link
//   stores mips->pc first, as IR_EXIT does. Without tail calls the links are plain calls, at most
//   kMaxLinkDepth deep. Anything else returns the PC to IRInterpret, as in M1.
// - Ops without a lowering run as interpreter islands: each maximal run of them is copied into a
//   buffer the module owns, ending in ExitToConst 1 (no PC is odd), and IRInterpret runs it. 1 means
//   go on; any other value is a PC to return (Break, Breakpoint, MemoryCheck, ValidateAddress*).
//
// Under the GPL-3.0-or-later, as the core it is built into (WasmEmitter.h comes from TabStationToo).

#include "pa_wasmjit.h"

#include <algorithm>
#include <bitset>
#include <cmath>
#include <cstddef>
#include <cstdio>
#include <cstring>
#include <deque>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include <emscripten.h>

#include "Common/CommonTypes.h"
#include "Common/Math/math_util.h"
#include "Core/Config.h"
#include "Core/ConfigValues.h"
#include "Core/Core.h"
#include "Core/CoreTiming.h"
#include "Core/HLE/HLE.h"
#include "Core/HLE/ReplaceTables.h"
#include "Core/MemFault.h"
#include "Core/MemMap.h"
#include "Core/MIPS/MIPS.h"
#include "Core/MIPS/MIPSTables.h"
#include "Core/MIPS/MIPSVFPUUtils.h"
#include "Core/MIPS/IR/IRInst.h"
#include "Core/MIPS/IR/IRInterpreter.h"
#include "Core/MIPS/IR/IRJit.h"
#include "WasmEmitter.h"

// From ppsspp-3039c4e-wasmjit.patch.
extern void (*g_paWasmJitHot)(const IRInst *inst);
extern u32 g_paArenaGen;
extern u32 g_paCallWasmCount;
extern u32 g_paBlockEntryCount;
extern MIPSComp::IRInterpreterJit *(*g_paCreateIRJit)(MIPSState *mips);

// pa_wasmjit.js.
extern "C" {
int pa_jit_js_init(void);
int pa_jit_table_grow(int n);
int pa_jit_load(const void *bytes, size_t len, int count, const int *slots);
void pa_jit_table_clear(int slot);
void pa_jit_table_copy(int dst, int src);
void pa_jit_error(char *buf, int len);
}

namespace PaWasmJit {
// Tests only (pa_jit_config 18): every so many syscalls from compiled code invalidate code, as a guest
// icache op would, while compiled code is on the stack.
void ChaosTick(MIPSState *mips);
}

// --- what compiled code calls ---------------------------------------------------------------------
// C linkage, scalar arguments, addresses taken: each keeps its table slot and exactly this signature
// under LTO, so a call_indirect to it from a generated module type-checks.
extern "C" {

u32 pa_jit_island(MIPSState *mips, const IRInst *run) {
	return IRInterpret(mips, run, nullptr);
}

// IRInterpreter.cpp's Syscall. HLE may switch threads, which rewrites every register.
void pa_jit_syscall(MIPSState *mips, u32 op) {
	RestoreHostRoundingMode();
	{
		Memory::IRInterpreterFaultScope outside(false);
		CallSyscall(mips, MIPSOpcode(op));
	}
	ApplyHostRoundingMode(mips);
	if (coreState != CORE_RUNNING_CPU)
		CoreTiming::ForceCheck(mips);
	PaWasmJit::ChaosTick(mips);
}

// IRInterpreter.cpp's SyscallUnresolved.
void pa_jit_syscall_unresolved(MIPSState *mips, u32 pc) {
	RestoreHostRoundingMode();
	{
		Memory::IRInterpreterFaultScope outside(false);
		CallSyscallUnresolvedAtPC(mips, pc);
	}
	ApplyHostRoundingMode(mips);
	if (coreState != CORE_RUNNING_CPU)
		CoreTiming::ForceCheck(mips);
}

// IRInterpreter.cpp's Interpret.
void pa_jit_interpret(MIPSState *mips, u32 op) {
	Memory::IRInterpreterFaultScope outside(false);
	MIPSInterpret(mips, MIPSOpcode(op));
}

// IRInterpreter.cpp's CallReplacement: writes r[dest] and charges the cycles.
void pa_jit_replacement(MIPSState *mips, u32 funcIndex, u32 dest) {
	const ReplacementTableEntry *f = GetReplacementFunc(funcIndex);
	RestoreHostRoundingMode();
	int cycles;
	{
		Memory::IRInterpreterFaultScope outside(false);
		cycles = f->replaceFunc(mips);
	}
	ApplyHostRoundingMode(mips);
	mips->r[dest] = cycles < 0 ? -1 : 0;
	mips->downcount -= cycles < 0 ? -cycles : cycles;
}

// The VFPU's own functions and the FPU's conversions, each a copy of its IRInterpreter.cpp handler with
// the operands as arguments: the same C on the same engine gives the same bits. None of them touches a
// register but its destination, so compiled code keeps its locals across them (but the FPRs written).
void pa_jit_vdot(MIPSState *mips, u32 dest, u32 src1, u32 src2) {
	mips->f[dest] = vfpu_dot(&mips->f[src1], &mips->f[src2]);
}
// vfpu_multi_dot's body (MIPSVFPUUtils.cpp at the pinned commit; its NEON path is ARM64's only) rather than
// a call to it: with a second caller, -flto stops inlining it into IRInterpret, and the interpreter, which
// players run by default, got a call more per Vec4VfpuMultiDot. vfpu_dot itself is shared, as before.
void pa_jit_vmultidot(MIPSState *mips, u32 dest, u32 src1, u32 src2, u32 constant) {
	const float *block = &mips->f[src1], *v = &mips->f[src2];
	const int terms = IRMULTIDOT_TERMS(constant);
	const bool transposed = (constant & IRMULTIDOT_TRANSPOSED) != 0;
	float result[4];
	for (int k = 0; k < 4; k++) {
		float a[4]{}, b[4]{};
		for (int i = 0; i < terms; i++) {
			a[i] = transposed ? block[4 * k + i] : block[4 * i + k];
			b[i] = v[i];
		}
		result[k] = vfpu_dot(a, b);
	}
	memcpy(&mips->f[dest], result, sizeof(result));
}
void pa_jit_fsin(MIPSState *mips, u32 dest, u32 src1) { mips->f[dest] = vfpu_sin(mips->f[src1]); }
void pa_jit_fcos(MIPSState *mips, u32 dest, u32 src1) { mips->f[dest] = vfpu_cos(mips->f[src1]); }
void pa_jit_frsqrt(MIPSState *mips, u32 dest, u32 src1) { mips->f[dest] = vfpu_rsqrt(mips->f[src1]); }
void pa_jit_frecip(MIPSState *mips, u32 dest, u32 src1) { mips->f[dest] = vfpu_rcp(mips->f[src1]); }
void pa_jit_fasin(MIPSState *mips, u32 dest, u32 src1) { mips->f[dest] = vfpu_asin(mips->f[src1]); }
void pa_jit_fvsqrt(MIPSState *mips, u32 dest, u32 src1) { mips->f[dest] = vfpu_sqrt(mips->f[src1]); }
void pa_jit_fexp2(MIPSState *mips, u32 dest, u32 src1) { mips->f[dest] = vfpu_exp2(mips->f[src1]); }
void pa_jit_flog2(MIPSState *mips, u32 dest, u32 src1) { mips->f[dest] = vfpu_log2(mips->f[src1]); }
void pa_jit_fsincos(MIPSState *mips, u32 dest, u32 src1) {
	float s, c;
	vfpu_sincos(mips->f[src1], s, c);
	mips->f[dest] = s;
	mips->f[dest + 1] = c;
}
void pa_jit_ffloattohalf2(MIPSState *mips, u32 dest, u32 src1, u32 src2) {
	mips->fi[dest] = vfpu_f2h(mips->fi[src1]) | ((u32)vfpu_f2h(mips->fi[src2]) << 16);
}
void pa_jit_fhalftofloat(MIPSState *mips, u32 dest, u32 src1, u32 src2) {
	mips->fi[dest] = vfpu_h2f((u16)(src2 ? mips->fi[src1] >> 16 : mips->fi[src1] & 0xFFFF));
}
void pa_jit_fround(MIPSState *mips, u32 dest, u32 src1) {
	mips->fs[dest] = SaturatedFloatToInt(round_ieee_754(FlushForConversion(mips, mips->f[src1])));
}
void pa_jit_ftrunc(MIPSState *mips, u32 dest, u32 src1) {
	mips->fs[dest] = SaturatedFloatToInt(truncf(FlushForConversion(mips, mips->f[src1])));
}
void pa_jit_fceil(MIPSState *mips, u32 dest, u32 src1) {
	mips->fs[dest] = SaturatedFloatToInt(ceilf(FlushForConversion(mips, mips->f[src1])));
}
void pa_jit_ffloor(MIPSState *mips, u32 dest, u32 src1) {
	mips->fs[dest] = SaturatedFloatToInt(floorf(FlushForConversion(mips, mips->f[src1])));
}
void pa_jit_fcvtws(MIPSState *mips, u32 dest, u32 src1) {
	float src = FlushForConversion(mips, mips->f[src1]);
	switch (IRRoundMode(mips->fcr31 & 3)) {
	case IRRoundMode::RINT_0: mips->fs[dest] = SaturatedFloatToInt(round_ieee_754(src)); break;
	case IRRoundMode::CAST_1: mips->fs[dest] = SaturatedFloatToInt(truncf(src)); break;
	case IRRoundMode::CEIL_2: mips->fs[dest] = SaturatedFloatToInt(ceilf(src)); break;
	case IRRoundMode::FLOOR_3: mips->fs[dest] = SaturatedFloatToInt(floorf(src)); break;
	}
}
void pa_jit_fcvtscaledws(MIPSState *mips, u32 dest, u32 src1, u32 src2) {
	float src = mips->f[src1];
	if (my_isnan(src)) {
		mips->fs[dest] = 2147483647L;
		return;
	}
	float mult = (float)(1UL << (src2 & 0x1F));
	double sv = src * mult;
	if (sv > (double)0x7fffffff) {
		mips->fs[dest] = 0x7fffffff;
	} else if (sv <= (double)(int)0x80000000) {
		mips->fs[dest] = 0x80000000;
	} else {
		switch (IRRoundMode(src2 >> 6)) {
		case IRRoundMode::RINT_0: mips->fs[dest] = (int)round_ieee_754(sv); break;
		case IRRoundMode::CAST_1: mips->fs[dest] = src >= 0 ? (int)floor(sv) : (int)ceil(sv); break;
		case IRRoundMode::CEIL_2: mips->fs[dest] = (int)ceil(sv); break;
		case IRRoundMode::FLOOR_3: mips->fs[dest] = (int)floor(sv); break;
		}
	}
}
// ApplyRoundingMode and RestoreRoundingMode: the host's mode is no register of the guest's.
void pa_jit_apply_rounding(MIPSState *mips) { ApplyHostRoundingMode(mips); }
void pa_jit_restore_rounding(MIPSState *mips) { RestoreHostRoundingMode(true); }

}  // extern "C"

namespace PaWasmJit {
namespace {

using WasmJit::ValType;

// --- MIPSState as the emitter addresses it ---------------------------------------------------------
// An IR GPR-class register r is the u32 at byte 4r (MIPS.h keeps r, f, v, t, vfpuCtrl, vt, pc, lo, hi
// in that order); an FPR f is the one at 4 (32 + f).
constexpr u32 OFF_PC = offsetof(MIPSState, pc);
constexpr u32 OFF_LO = offsetof(MIPSState, lo);
constexpr u32 OFF_DC = offsetof(MIPSState, downcount);
constexpr u32 OFF_LLBIT = offsetof(MIPSState, llBit);
static_assert(offsetof(MIPSState, r) == 0, "r at 0");
static_assert(offsetof(MIPSState, f) == 4 * 32, "f after r");
static_assert(offsetof(MIPSState, vfpuCtrl) == 4 * IRREG_VFPU_CTRL_BASE, "vfpuCtrl at 208");
static_assert(OFF_LO == 4 * IRREG_LO && offsetof(MIPSState, hi) == 4 * IRREG_HI, "lo, hi at 242, 243");
static_assert(OFF_LO % 8 == 0, "lo and hi as one i64");
static_assert(OFF_PC == 4 * 241, "pc at 241");

constexpr u32 GprOff(u32 r) { return 4 * r; }
constexpr u32 FprOff(u32 f) { return 4 * (32 + f); }

// The registers that get a local: the GPRs, the IR temps and VFPU control words, lo, hi and fpcond. No C
// helper but those that get every register stored first (syscalls, Interpret, replacements, islands)
// reads one; fcr31, which the rounding helpers read, stays in memory.
inline bool Cacheable(u32 r) {
	return r < 32 || (r >= 192 && r < 224) || r == IRREG_LO || r == IRREG_HI || r == IRREG_FPCOND;
}

// --- tuning (pa_jit_config) -------------------------------------------------------------------------
struct Config {
	int thresh = 128;          // entries before a block is hot (TabStationToo's JitHeat)
	int batch = 64;            // blocks a module, at most
	// Compile a block when its native ops >= islandFactor * its island runs. On the race in WebKit, 0, 4,
	// 6 and 8 ran within the machine's noise of each other and 2 slower (README "The JIT"); 6 compiles 11%
	// fewer blocks than 0, and compiled blocks are the session's budget (§16.3.6).
	int islandFactor = 6;
	int maxBlocks = 30000;     // blocks compiled in a session, at most (§16.3.6)
	int budgetMaxUs = 1500;    // compile time a frame: clamp(14 ms - the frame's own, 0.2, this)
	int maxModules = 4;        // modules a frame, at most
	int canary = 1;            // time fresh code now and then, and stop compiling if it got 5x slower
	int canaryWindow = 300;    // frames between canaries: two thirds warming it up, then up to 3 timings
	int selftestWarm = 0;      // the fuzz: calls each compiled block gets on a scratch state before the one compared
	int selftestTwoNans = 0;   // the fuzz: half its adds and multiplies get NaNs on both sides (V8 only: PlantNoNan)
	int selftestChain = 0;     // the fuzz: programs of blocks that link to each other, on the real JIT
	// M3.
	int links = 15;            // which links: 1 loops, 2 direct calls in a module, 4 link slots, 8 ExitToReg lookups
	int tailCalls = 1;         // links as tail calls where the engine has them (else bounded calls)
	int writeBack = 1;         // registers written back on the way out (else through, at each write)
	int foldConst = 1;         // constant registers folded into addresses and exits
	int countLinks = 0;        // compiled code counts the links it takes (pa_jit_stats "links")
	int chaos = 0;             // tests only: every this many syscalls from compiled code invalidate code
};
Config g_cfg;

enum ConfigKey {
	CFG_THRESH = 1,
	CFG_BATCH = 2,
	CFG_ISLAND_FACTOR = 3,
	CFG_MAX_BLOCKS = 4,
	CFG_BUDGET_MAX_US = 5,
	CFG_MAX_MODULES = 6,
	CFG_CANARY = 7,
	CFG_CANARY_WINDOW = 8,
	CFG_TRIP_CANARY = 9,  // tests only: stop compiling now, as the canary does on its second slow window
	// Tests only: the fuzz calls each compiled block this many times first (on a scratch state), so the
	// call it compares runs in the engine's optimising tier, not the first one (JSC tiers up after ~10k).
	CFG_SELFTEST_WARM = 10,
	// Tests only, and only in V8: half the fuzz's adds and multiplies get a NaN in every lane of both
	// sources, so an operand order swapped in a lowering shows (FuzzGen::PlantNoNan says why not in JSC).
	CFG_SELFTEST_TWO_NANS = 11,
	// Tests only: pa_jit_selftest runs programs of blocks that link to each other on a real IRWasmJit.
	CFG_SELFTEST_CHAIN = 12,
	// M3: which links (LINK_* bits), tail calls, write-back, constant folding, link counters.
	CFG_LINKS = 13,
	CFG_TAIL_CALLS = 14,
	CFG_WRITE_BACK = 15,
	CFG_FOLD_CONST = 16,
	CFG_COUNT_LINKS = 17,
	// Tests only: every N syscalls from compiled code invalidate code (ChaosTick); clear the cache now.
	CFG_TEST_CHAOS = 18,
	CFG_TEST_CLEAR = 19,
};

// --- counters (pa_jit_stats) ------------------------------------------------------------------------
struct OpCount {
	u64 blocks = 0;  // static: blocks (or island runs) it counts in
	u64 exec = 0;    // dynamic: executions seen
};
struct Stats {
	u64 blocks = 0, modules = 0, bytes = 0;
	u64 rejected = 0, rejectedNoExit = 0;
	u64 invalidated = 0, clears = 0, flushes = 0, freedSlots = 0;
	u64 rangeErrors = 0, loadErrors = 0;
	u64 instances = 0, reverted = 0;
	u64 chaos = 0;
	double compileMs = 0, compileMsMax = 0;
	u64 framesCompiling = 0, frames = 0;
	u32 callWasmBase = 0, blockEntryBase = 0;
	OpCount islands[256];
	OpCount rejectedBy[256];
	double canaryBaseNs = 0, canaryLastNs = 0;
	u64 canaryWindows = 0, canaryTimings = 0, canaryStrikes = 0;
	std::string lastError;
};
Stats g_stats;
// Island runs executed, by their first op: compiled code adds to these itself.
u32 g_islandExec[256];

double Now() { return emscripten_get_now(); }

// --- what compiled code reads and counts at fixed addresses (M3) ---------------------------------------
// The kinds of link (pa_jit_config 13).
enum : u32 {
	LINK_SELF = 1,    // an exit to the block's own start: a wasm loop
	LINK_DIRECT = 2,  // a constant exit to a block of the same module: a direct tail call
	LINK_TABLE = 4,   // any other constant exit: a tail call through the target's link slot
	LINK_REG = 8,     // ExitToReg and ExitToPC: the shadow and the arena head looked up inline
};
// Link counters (pa_jit_config 17): what compiled code adds to when it takes a link, the stub when a link
// slot had no compiled block behind it, a lookup when it found none, and a compiled block each time it is
// entered (from IRInterpret's CallWasm or a link: entries = callWasm + direct + table - stub + reg).
enum { LC_SELF, LC_DIRECT, LC_TABLE, LC_STUB, LC_REG, LC_REGMISS, LC_ENTER, LC_COUNT };
u32 g_linkCount[LC_COUNT];
// The live JIT's arena_.data(), for ExitToReg's lookup of a CallWasm head; kept current wherever the arena
// can move (CompileTargetBlock, ClearCache).
u32 g_arenaCell = 0;
// Bumped whenever every block goes at once (ClearCache, a JIT made or deleted), when arena offsets start
// over: a direct link checks it beside the shadow entry.
u32 g_epochCell = 1;
// Links that are plain calls (no tail calls) nested now, and the most there may be.
u32 g_depthCell = 0;
constexpr u32 kMaxLinkDepth = 32;

u32 Cell(const void *p) { return (u32)(uintptr_t)p; }

// --- the function table: slots of our own ------------------------------------------------------------
// Pre-grown in 64k chunks with a free list of our own, and filled with setWasmTableEntry (which keeps
// Emscripten's mirror coherent), not addFunction: that grows the table by one each call, O(table) in
// JSC (§16.3.5).
struct ModuleRec {
	std::vector<int> slots;
	int live = 0;
	std::unique_ptr<IRInst[]> islands;  // the island runs its code points at
};

struct Runtime {
	bool tried = false, ready = false, tailCalls = false;
	int stubSlot = 0;  // the link slots' stub: returns mips->pc (which the link stored), counting LC_STUB
	std::vector<int> freeSlots;
	std::unordered_map<int, int> slotModule;
	std::unordered_map<int, ModuleRec> modules;
	int nextModule = 1;

	bool Init() {
		if (tried)
			return ready;
		tried = true;
		tailCalls = pa_jit_js_init() == 1;
		ready = Grow();
		if (ready)
			stubSlot = CompileStub();
		return ready;
	}
	// The stub a link slot holds while the block it stands for is not compiled: as if the linking block
	// had returned the target itself (the link stored it in mips->pc), so IRInterpret's IR_EXIT goes on.
	int CompileStub() {
		WasmJit::Module m;
		const u32 type = m.type({ValType::I32}, {ValType::I32});
		m.import_memory_unbounded("e", "m", 0);
		m.import_table("e", "t", 0);
		WasmJit::Function f(1);
		const u32 count = Cell(&g_linkCount[LC_STUB]);
		f.i32_const(0);
		f.i32_const(0);
		f.i32_load(count);
		f.i32_const(1);
		f.i32_add();
		f.i32_store(count);
		f.local_get(0);
		f.i32_load(offsetof(MIPSState, pc));
		m.export_function("b0", m.add_function(type, f));
		const std::vector<u8> bytes = m.finish();
		std::vector<int> slot;
		if (!Alloc(1, slot))
			return 0;
		if (pa_jit_load(bytes.data(), bytes.size(), 1, slot.data()) != 1) {
			freeSlots.push_back(slot[0]);
			return 0;
		}
		return slot[0];
	}
	bool Grow() {
		const int first = pa_jit_table_grow(65536);
		if (first < 0)
			return false;
		for (int s = first + 65535; s >= first; --s)
			freeSlots.push_back(s);
		return true;
	}
	bool Alloc(int n, std::vector<int> &out) {
		out.clear();
		while ((int)freeSlots.size() < n) {
			if (!Grow())
				return false;
		}
		for (int i = 0; i < n; ++i) {
			out.push_back(freeSlots.back());
			freeSlots.pop_back();
		}
		return true;
	}
	// Only where no code of the module can be on the stack.
	void Release(int slot) {
		pa_jit_table_clear(slot);
		freeSlots.push_back(slot);
		g_stats.freedSlots++;
		auto it = slotModule.find(slot);
		if (it == slotModule.end())
			return;
		const int id = it->second;
		slotModule.erase(it);
		auto mod = modules.find(id);
		if (mod != modules.end() && --mod->second.live <= 0)
			modules.erase(mod);
	}
};
Runtime g_rt;

// --- what to do with each op --------------------------------------------------------------------------
enum class Kind : u8 { Native, Helper, Island, Reject };

bool IsUnconditionalExit(IROp op) {
	switch (op) {
	case IROp::ExitToConst:
	case IROp::ExitToReg:
	case IROp::ExitToPC:
		return true;
	default:
		return false;
	}
}

bool IsElseExit(IROp op) {
	return op >= IROp::OptExitToConstIfEqElse && op <= IROp::OptExitToConstIfLeZElse;
}

Kind Classify(IROp op, bool plainLoad16) {
	switch (op) {
	case IROp::SetConst: case IROp::SetConstF: case IROp::Mov:
	case IROp::Add: case IROp::Sub: case IROp::Neg: case IROp::Not:
	case IROp::And: case IROp::Or: case IROp::Xor:
	case IROp::AddConst: case IROp::OptAddConst: case IROp::SubConst:
	case IROp::AndConst: case IROp::OrConst: case IROp::XorConst: case IROp::OptAndConst: case IROp::OptOrConst:
	case IROp::Shl: case IROp::Shr: case IROp::Sar: case IROp::Ror:
	case IROp::ShlImm: case IROp::ShrImm: case IROp::SarImm: case IROp::RorImm:
	case IROp::Slt: case IROp::SltConst: case IROp::SltU: case IROp::SltUConst:
	case IROp::Clz: case IROp::MovZ: case IROp::MovNZ: case IROp::Max: case IROp::Min:
	case IROp::BSwap16: case IROp::BSwap32: case IROp::Ext8to32: case IROp::Ext16to32:
	case IROp::MtLo: case IROp::MtHi: case IROp::MfLo: case IROp::MfHi:
	case IROp::Mult: case IROp::MultU: case IROp::Madd: case IROp::MaddU: case IROp::Msub: case IROp::MsubU:
	case IROp::Load8: case IROp::Load8Ext: case IROp::Load32: case IROp::Load32Linked:
	case IROp::LoadFloat: case IROp::LoadVec4:
	case IROp::Store8: case IROp::Store32: case IROp::Store32Conditional: case IROp::StoreFloat: case IROp::StoreVec4:
	case IROp::FMov:
	// M2: the FPU, Vec4 and the VFPU (its packs and unpacks too), Div/DivU, ReverseBits.
	case IROp::Div: case IROp::DivU:
	case IROp::FAdd: case IROp::FSub: case IROp::FMul: case IROp::FDiv: case IROp::FMin: case IROp::FMax:
	case IROp::FSqrt: case IROp::FNeg: case IROp::FAbs: case IROp::FSign: case IROp::FSat0_1: case IROp::FSatMinus1_1:
	case IROp::FCvtSW: case IROp::FCvtScaledSW:
	case IROp::FRound: case IROp::FTrunc: case IROp::FCeil: case IROp::FFloor: case IROp::FCvtWS: case IROp::FCvtScaledWS:
	case IROp::FMovFromGPR: case IROp::FMovToGPR: case IROp::OptFCvtSWFromGPR: case IROp::OptFMovToGPRShr8:
	case IROp::FpCondFromReg: case IROp::FpCondToReg: case IROp::FpCtrlFromReg: case IROp::FpCtrlToReg:
	case IROp::VfpuCtrlToReg: case IROp::SetCtrlVFPU: case IROp::SetCtrlVFPUReg: case IROp::SetCtrlVFPUFReg:
	case IROp::FCmp: case IROp::FCmpVfpuBit: case IROp::FCmpVfpuAggregate: case IROp::FCmovVfpuCC:
	case IROp::Vec4Init: case IROp::Vec4Shuffle: case IROp::Vec4Blend: case IROp::Vec4Mov:
	case IROp::Vec4Add: case IROp::Vec4Sub: case IROp::Vec4Mul: case IROp::Vec4Div: case IROp::Vec4Scale:
	case IROp::Vec4Neg: case IROp::Vec4Abs: case IROp::Vec4Min: case IROp::Vec4Max: case IROp::Vec4Slt: case IROp::Vec4Sge:
	case IROp::Vec4CvtScaledSW:
	case IROp::Vec2Unpack16To31: case IROp::Vec2Unpack16To32: case IROp::Vec4Unpack8To32: case IROp::Vec4DuplicateUpperBitsAndShift1:
	case IROp::Vec4Pack31To8: case IROp::Vec4Pack32To8: case IROp::Vec2Pack31To16: case IROp::Vec2Pack32To16:
	case IROp::ReverseBits:
	case IROp::Vec4VfpuDot: case IROp::Vec4VfpuMultiDot:
	case IROp::FSin: case IROp::FCos: case IROp::FRSqrt: case IROp::FRecip: case IROp::FAsin: case IROp::FVSqrt:
	case IROp::FExp2: case IROp::FLog2: case IROp::FSinCos: case IROp::FFloatToHalf2: case IROp::FHalfToFloat:
	case IROp::ApplyRoundingMode: case IROp::RestoreRoundingMode: case IROp::UpdateRoundingMode:
	case IROp::Downcount:
	case IROp::SetPC: case IROp::SetPCConst:
	case IROp::ExitToConst: case IROp::ExitToReg: case IROp::ExitToPC:
	case IROp::ExitToConstIfEq: case IROp::ExitToConstIfNeq: case IROp::ExitToConstIfGtZ:
	case IROp::ExitToConstIfGeZ: case IROp::ExitToConstIfLtZ: case IROp::ExitToConstIfLeZ:
	case IROp::OptExitToConstIfEqElse: case IROp::OptExitToConstIfNeqElse: case IROp::OptExitToConstIfGtZElse:
	case IROp::OptExitToConstIfGeZElse: case IROp::OptExitToConstIfLtZElse: case IROp::OptExitToConstIfLeZElse:
		return Kind::Native;
	// With the software renderer VRAM's depth mirrors remap 16-bit accesses (DepthMirrored16), so those
	// go to the interpreter; with GL the mirrors change nothing and the access is plain.
	case IROp::Load16: case IROp::Load16Ext: case IROp::Store16:
		return plainLoad16 ? Kind::Native : Kind::Island;
	case IROp::Syscall: case IROp::SyscallUnresolved: case IROp::Interpret: case IROp::CallReplacement:
		return Kind::Helper;
	// Never in an interpreter block (the frontend makes no Fp exits for it; Nop and Bad are unreachable).
	case IROp::ExitToConstIfFpTrue: case IROp::ExitToConstIfFpFalse:
	case IROp::Nop: case IROp::Bad: case IROp::BlockEntry: case IROp::CallWasm:
		return Kind::Reject;
	default:
		return Kind::Island;
	}
}

// Classify, and the operands the lowering can't take: those go to the interpreter.
Kind ClassifyInst(const IRInst &in, IROp op, bool plainLoad16) {
	const Kind kind = Classify(op, plainLoad16);
	// vec4InitValues has 8 rows (IRInterpreter.cpp); no frontend writes more.
	if (op == IROp::Vec4Init && in.src1 >= 8)
		return Kind::Island;
	return kind;
}

const char *OpName(IROp op) {
	const IRMeta *meta = GetIRMeta(op);
	return meta ? meta->name : "?";
}

// One block, as it will be compiled.
struct Plan {
	const IRInst *ir = nullptr;  // ir[0] is the leading Downcount, or the op that stands in for it
	int n = 0;                   // ops emitted, through the block's last exit
	u32 dc0 = 0;                 // the leading Downcount's amount
	std::vector<std::pair<int, int>> runs;  // island runs: first op, length
	int nativeOps = 0, islandOps = 0;
	IROp rejectOp = IROp::Nop;
	bool noExit = false;
	// Filled in by whoever compiles it: the block's arena offset and guest address (0 when it is no block of
	// the arena's: the canary, the plain fuzz; it then takes no links).
	int entry = 0;
	u32 start = 0;
};

void MergeIslands(Plan &p);

// Plans a block of `count` IR ops; false if it can't be compiled (rejectOp, noExit say why).
bool MakePlan(const IRInst *ir, int count, bool plainLoad16, Plan &p) {
	p.ir = ir;
	p.dc0 = ir[0].constant;
	p.runs.clear();
	p.nativeOps = p.islandOps = 0;
	int runStart = -1;
	for (int i = 0; i < count; ++i) {
		const IROp op = i == 0 ? IROp::Downcount : ir[i].op;
		const Kind kind = ClassifyInst(ir[i], op, plainLoad16);
		if (kind == Kind::Reject) {
			p.rejectOp = op;
			return false;
		}
		if (kind == Kind::Island) {
			if (runStart < 0)
				runStart = i;
			p.islandOps++;
			continue;
		}
		if (runStart >= 0) {
			p.runs.emplace_back(runStart, i - runStart);
			runStart = -1;
		}
		p.nativeOps++;
		if (IsElseExit(op)) {
			// The ExitToConst after it only holds the other target; the interpreter never runs it.
			if (i + 1 >= count || ir[i + 1].op != IROp::ExitToConst)
				break;
			p.n = i + 2;
			MergeIslands(p);
			return true;
		}
		if (IsUnconditionalExit(op)) {
			p.n = i + 1;
			MergeIslands(p);
			return true;
		}
	}
	// It would run off its end into whatever follows in the arena.
	p.noExit = true;
	return false;
}

// Two island runs with at most kIslandGap plain native ops between them become one run: entering
// IRInterpret costs more than interpreting a few ops. Exits, Downcounts and helpers are never taken in
// (the interpreter would charge, return or call out where the compiled code expects not to).
constexpr int kIslandGap = 3;

bool Absorbable(const IRInst &in) {
	const IROp op = in.op;
	return op != IROp::Downcount && !IsUnconditionalExit(op) && !IsElseExit(op) && !(op >= IROp::ExitToConstIfEq && op <= IROp::ExitToConstIfLeZ) &&
		ClassifyInst(in, op, true) == Kind::Native;
}

void MergeIslands(Plan &p) {
	if (p.runs.size() < 2)
		return;
	std::vector<std::pair<int, int>> merged{p.runs[0]};
	for (size_t i = 1; i < p.runs.size(); ++i) {
		auto &last = merged.back();
		const int gapStart = last.first + last.second, gapEnd = p.runs[i].first;
		bool absorb = gapEnd - gapStart <= kIslandGap;
		for (int k = gapStart; absorb && k < gapEnd; ++k)
			absorb = Absorbable(p.ir[k]);
		if (absorb) {
			p.nativeOps -= gapEnd - gapStart;
			p.islandOps += gapEnd - gapStart;
			last.second = p.runs[i].first + p.runs[i].second - last.first;
		} else {
			merged.push_back(p.runs[i]);
		}
	}
	p.runs = std::move(merged);
}

// --- the block compiler -------------------------------------------------------------------------------
// The block shadow's index mask on wasm32 (MemMap.h): one u32 a guest word over 256 MB.
constexpr u32 kShadowMask = Memory::BLOCK_SHADOW_ENTRY_MASK;
static_assert(Memory::BLOCK_SHADOW_ENTRY_MASK == 0x0FFFFFFC, "the block shadow of a 32-bit build");

// Hands out the table slot that stands for a constant target (one per block shadow entry, see
// IRWasmJit::SlotFor); <= 0 when there is none. A plain function and its context, not a base class: a
// second base would move IRWasmJit's members, which IRInterpret's BlockEntry reads where -flto inlines
// IRWasmJit::Hot (see SelfTestChain's hook).
struct Linker {
	int (*slotFor)(void *self, u32 target) = nullptr;
	void *self = nullptr;
	int SlotFor(u32 target) const { return slotFor(self, target); }
};

struct CompileCtx {
	u32 memBase = 0;          // the address space's base: the memarg offset of every guest access
	bool plainLoad16 = true;  // see Classify
	bool writeBack = true;    // registers written back (else through)
	bool foldConst = true;    // constant registers folded into addresses and exits
	// Links (LINK_* bits; 0 for none: the canary, the plain fuzz) and what they need.
	u32 links = 0;
	bool tailCalls = false;
	bool countLinks = false;
	const Memory::AddressSpace *space = nullptr;  // which constant targets are valid
	u32 shadowBase = 0;                           // space->blockShadow
	Linker linker;
};

struct Types {
	u32 block, island, v1, v2, v3, v4, v5;  // vN: N i32 parameters, no result
};

// The blocks of the module being compiled, by block shadow entry: an exit there is a direct call.
struct InModule {
	u32 func;   // function index in the module
	int entry;  // arena offset
};
typedef std::unordered_map<u32, InModule> BatchMap;

// What the compiler knows of the registers at a point of the code.
struct RegState {
	std::bitset<256> loaded;  // the local holds the register's value
	std::bitset<256> dirty;   // ... newer than MIPSState's (dirty implies loaded)
	std::bitset<256> known;   // ... and it is kval (known implies loaded)
	u32 kval[256] = {};
	bool dcLoaded = false, dcDirty = false;  // the same for the downcount
};

class BlockCompiler {
public:
	BlockCompiler(WasmJit::Function &f, const CompileCtx &ctx, const Types &types, const BatchMap *batch)
		: f_(f), ctx_(ctx), types_(types), batch_(batch) {
		std::fill(std::begin(local_), std::end(local_), -1);
	}

	// Whether one of the block's constant exits is its own start (a loop, when LINK_SELF).
	static bool HasSelfExit(const Plan &p) {
		if (!p.start)
			return false;
		for (int i = 1; i < p.n; ++i) {
			const IROp op = p.ir[i].op;
			if (op == IROp::ExitToConst || (op >= IROp::ExitToConstIfEq && op <= IROp::ExitToConstIfLeZ)) {
				if (p.ir[i].constant == p.start)
					return true;
			} else if (IsElseExit(op)) {
				if (p.ir[i].constant == p.start || (i + 1 < p.n && p.ir[i + 1].constant == p.start))
					return true;
				++i;
			}
		}
		return false;
	}

	// top: the state at the top of the block's loop when it is a self-loop, else null.
	void Emit(const Plan &p, const std::vector<const IRInst *> &runCopies, const RegState *top) {
		plan_ = &p;
		mayInvalidate_ = MayInvalidate(p);
		Count(LC_ENTER);
		if (top) {
			// What the loop's top keeps in locals is loaded once before it.
			loop_ = true;
			top_ = *top;
			for (u32 r = 0; r < 256; ++r) {
				if (top_.loaded[r])
					LoadInto(r);
			}
			if (top_.dcLoaded) {
				f_.local_get(0);
				f_.i32_load(OFF_DC);
				f_.local_set(DcLocal());
			}
			f_.loop();
			loopLabel_ = f_.depth();
			s_ = top_;
		}
		size_t run = 0;
		for (int i = 0; i < p.n;) {
			if (run < p.runs.size() && p.runs[run].first == i) {
				EmitIsland(runCopies[run], p.ir[i].op);
				i += p.runs[run].second;
				run++;
				continue;
			}
			const IRInst &inst = p.ir[i];
			const IROp op = i == 0 ? IROp::Downcount : inst.op;
			if (IsElseExit(op)) {
				EmitElse(op, inst, p.ir[i + 1].constant);
				i += 2;
				continue;
			}
			EmitOp(op, inst);
			i++;
		}
		if (loop_) {
			// The body always leaves by a return or a branch to the top.
			f_.end();
			f_.unreachable();
		}
	}

	// After a first pass over a self-loop: the state its back edges reach the top with (loaded on all of
	// them, dirty on any), or nothing if none was reached.
	bool SawBackEdge() const { return backEdges_ > 0; }
	const RegState &BackEdges() const { return meet_; }

private:
	// Whether the block calls C that can invalidate code (and so itself) while it runs: then a loop checks
	// its own shadow entry before going round again. Interpret runs the guest's icache op; a syscall,
	// a replacement or an island can reach anything.
	static bool MayInvalidate(const Plan &p) {
		if (!p.runs.empty())
			return true;
		for (int i = 1; i < p.n; ++i) {
			switch (p.ir[i].op) {
			case IROp::Syscall: case IROp::SyscallUnresolved: case IROp::Interpret: case IROp::CallReplacement:
				return true;
			default:
				break;
			}
		}
		return false;
	}

	// --- registers ---
	u32 LocalFor(u32 r) {
		if (local_[r] < 0)
			local_[r] = (int)f_.local(ValType::I32);
		return (u32)local_[r];
	}
	u32 DcLocal() {
		if (dcLocal_ < 0)
			dcLocal_ = (int)f_.local(ValType::I32);
		return (u32)dcLocal_;
	}
	void LoadInto(u32 r) {
		f_.local_get(0);
		f_.i32_load(GprOff(r));
		f_.local_set(LocalFor(r));
	}
	void GetReg(u32 r) {
		if (!Cacheable(r)) {
			f_.local_get(0);
			f_.i32_load(GprOff(r));
			return;
		}
		const u32 l = LocalFor(r);
		if (s_.loaded[r]) {
			f_.local_get(l);
		} else {
			f_.local_get(0);
			f_.i32_load(GprOff(r));
			f_.local_tee(l);
			s_.loaded[r] = true;
		}
	}
	// A register write: BeginSet, push the value, EndSet (then Known, if the value is a constant).
	void BeginSet(u32 r) {
		if (!Cacheable(r))
			f_.local_get(0);
	}
	void EndSet(u32 r) {
		if (!Cacheable(r)) {
			f_.i32_store(GprOff(r));
			return;
		}
		const u32 l = LocalFor(r);
		f_.local_set(l);
		s_.loaded[r] = true;
		s_.known[r] = false;
		if (ctx_.writeBack) {
			s_.dirty[r] = true;
		} else {
			f_.local_get(0);
			f_.local_get(l);
			f_.i32_store(GprOff(r));
		}
	}
	void Known(u32 r, bool known, u32 value) {
		if (!Cacheable(r) || !ctx_.foldConst)
			return;
		s_.known[r] = known;
		s_.kval[r] = value;
	}
	bool KnownValue(u32 r, u32 &value) const {
		if (!ctx_.foldConst || !Cacheable(r) || !s_.known[r])
			return false;
		value = s_.kval[r];
		return true;
	}
	void Flush(u32 r) {
		f_.local_get(0);
		f_.local_get(LocalFor(r));
		f_.i32_store(GprOff(r));
		s_.dirty[r] = false;
	}
	// Every dirty register and the downcount to MIPSState: before leaving, or calling C that reads them.
	void FlushAll() {
		for (u32 r = 0; r < 256; ++r) {
			if (s_.dirty[r])
				Flush(r);
		}
		if (s_.dcDirty) {
			f_.local_get(0);
			f_.local_get(DcLocal());
			f_.i32_store(OFF_DC);
			s_.dcDirty = false;
		}
	}
	// After C that may have written any register or the downcount.
	void UnloadAll() {
		s_.loaded.reset();
		s_.dirty.reset();
		s_.known.reset();
		s_.dcLoaded = s_.dcDirty = false;
	}
	// Around calls to C that read and write MIPSState (Syscall, Interpret, CallReplacement, islands).
	void BeforeCallOut() {
		ChargePending();
		FlushAll();
	}
	void AfterCallOut() { UnloadAll(); }
	// FPR f lies where GPR-class register 32 + f lives (192..223 are FPRs 160..191). An FPR read must see a
	// value a local of that register holds dirty; an FPR write leaves the local stale.
	void FprRead(u32 f, u32 count) {
		for (u32 k = 0; k < count; ++k) {
			const u32 r = 32 + f + k;
			if (r < 256 && s_.dirty[r])
				Flush(r);
		}
	}
	void FprWritten(u32 f, u32 count) {
		for (u32 k = 0; k < count; ++k) {
			const u32 r = 32 + f + k;
			if (r < 256) {
				s_.loaded[r] = false;
				s_.dirty[r] = false;
				s_.known[r] = false;
			}
		}
	}
	// i32 scratch locals: 0 an island's result; 1-3 Store32Conditional's; 4-7 within one M2 lowering (Div,
	// FMul, FMin/FMax, FSign, the CC aggregate, the packs, ReverseBits); 8-10 the exits' (the target, a head
	// word or slot, a call's result).
	u32 Scratch(int which) {
		if (scratch_[which] < 0)
			scratch_[which] = (int)f_.local(ValType::I32);
		return (u32)scratch_[which];
	}
	u32 ScratchV(int which) {
		if (scratchV_[which] < 0)
			scratchV_[which] = (int)f_.local(ValType::V128);
		return (u32)scratchV_[which];
	}
	u32 ScratchI64() {
		if (scratchI64_ < 0)
			scratchI64_ = (int)f_.local(ValType::I64);
		return (u32)scratchI64_;
	}

	// --- FPRs and vectors, always in MIPSState ---
	void LoadF(u32 f) { FprRead(f, 1); f_.local_get(0); f_.f32_load(FprOff(f)); }
	void LoadFI(u32 f) { FprRead(f, 1); f_.local_get(0); f_.i32_load(FprOff(f)); }
	void LoadV(u32 f) { FprRead(f, 4); f_.local_get(0); f_.simd_mem(WasmJit::Simd::V128Load, FprOff(f), 2); }
	// An FPR write: local 0, the value, then one of these.
	void StoreF(u32 f) { f_.f32_store(FprOff(f)); FprWritten(f, 1); }
	void StoreFI(u32 f) { f_.i32_store(FprOff(f)); FprWritten(f, 1); }
	void StoreV(u32 f) { f_.simd_mem(WasmJit::Simd::V128Store, FprOff(f), 2); FprWritten(f, 4); }
	void ConstV(u32 a, u32 b, u32 c, u32 d) { f_.v128_const_i32x4(a, b, c, d); }
	void ConstV(u32 a) { f_.v128_const_i32x4(a, a, a, a); }
	void Simd(WasmJit::Simd op) { f_.simd(op); }
	// FPR f op FPR g, written to FPR d.
	void FBin(const IRInst &in, void (WasmJit::Function::*op)()) {
		f_.local_get(0);
		LoadF(in.src1);
		LoadF(in.src2);
		(f_.*op)();
		StoreF(in.dest);
	}
	void VBin(const IRInst &in, WasmJit::Simd op) {
		f_.local_get(0);
		LoadV(in.src1);
		LoadV(in.src2);
		Simd(op);
		StoreV(in.dest);
	}
	// The bits of an f32 on the stack: my_isnan, my_isinf, my_isnanorinf (math_util.h) as integer tests.
	void IsNanBits() { f_.i32_const(0x7FFFFFFF); f_.i32_and(); f_.i32_const(0x7F800000); f_.i32_gt_u(); }
	void IsInfBits() { f_.i32_const(0x7FFFFFFF); f_.i32_and(); f_.i32_const(0x7F800000); f_.i32_eq(); }
	void IsNanOrInfBits() { f_.i32_const(0x7F800000); f_.i32_and(); f_.i32_const(0x7F800000); f_.i32_eq(); }
	// vfpu_minmax_key (MIPSVFPUUtils.h) of the bits in local l.
	void MinMaxKey(u32 l) {
		f_.local_get(l);
		f_.local_get(l); f_.i32_const(31); f_.i32_shr_s(); f_.i32_const(1); f_.i32_shr_u();
		f_.i32_xor();
		f_.i32_const(0);
		f_.local_get(l); f_.i32_const(0x7F800000); f_.i32_and();
		f_.select();
	}
	// The same on four lanes in v128 local l, as Emscripten compiles Vec4Min's loop: the key where the
	// exponent is not zero, else 0.
	void MinMaxKeyV(u32 l) {
		f_.local_get(l);
		f_.local_get(l); f_.i32_const(31); Simd(WasmJit::Simd::I32x4ShrS); f_.i32_const(1); Simd(WasmJit::Simd::I32x4ShrU);
		Simd(WasmJit::Simd::V128Xor);
		f_.local_get(l); ConstV(0x7F800000); Simd(WasmJit::Simd::V128And);
		ConstV(0); Simd(WasmJit::Simd::I32x4Ne);
		Simd(WasmJit::Simd::V128And);
	}
	// A call to one of the C wrappers above with (mips, then these constants).
	void Call(void *fn, u32 type, std::initializer_list<u32> args) {
		f_.local_get(0);
		for (u32 a : args)
			f_.i32_const((s32)a);
		CallHelper((u32)(uintptr_t)fn, type);
	}
	// One of the VFPU and FPU wrappers: they read the FPR spans in `reads` ({first, count}) and write `count`
	// from dest, nothing else. What a local holds dirty under the FPRs read is stored first, and nothing else.
	void CallFpr(void *fn, u32 type, std::initializer_list<u32> args, std::initializer_list<std::pair<u32, u32>> reads, u32 dest,
		u32 count) {
		for (const auto &span : reads)
			FprRead(span.first, span.second);
		Call(fn, type, args);
		FprWritten(dest, count);
	}

	// --- downcount: a local, charged with what is pending where it is read ---
	void EnsureDc() {
		if (s_.dcLoaded)
			return;
		f_.local_get(0);
		f_.i32_load(OFF_DC);
		f_.local_set(DcLocal());
		s_.dcLoaded = true;
	}
	void ChargePending() {
		if (pendingDc_ == 0)
			return;
		EnsureDc();
		f_.local_get(DcLocal());
		f_.i32_const((s32)pendingDc_);
		f_.i32_sub();
		f_.local_set(DcLocal());
		pendingDc_ = 0;
		if (ctx_.writeBack) {
			s_.dcDirty = true;
		} else {
			f_.local_get(0);
			f_.local_get(DcLocal());
			f_.i32_store(OFF_DC);
		}
	}
	// Pushes downcount >= 0, IR_EXIT's condition to chain (what is pending charged first).
	void DcNotNegative() {
		EnsureDc();
		f_.local_get(DcLocal());
		f_.i32_const(0);
		f_.i32_ge_s();
	}

	// --- guest memory ---
	// Pushes a guest access's address and returns the memarg offset to use with it: base + ((rs + C) &
	// mask), or with rs a known constant, all of it as the offset over an address of 0.
	u32 Addr(u32 rs, u32 c) {
		u32 k;
		if (KnownValue(rs, k)) {
			const u64 at = (u64)ctx_.memBase + ((k + c) & Memory::MEMVIEW32_MASK);
			if (at <= 0xFFFFFFFFull) {
				f_.i32_const(0);
				return (u32)at;
			}
		}
		GetReg(rs);
		if (c != 0) {
			f_.i32_const((s32)c);
			f_.i32_add();
		}
		f_.i32_const((s32)Memory::MEMVIEW32_MASK);
		f_.i32_and();
		return ctx_.memBase;
	}

	// --- pieces ---
	void Bin(const IRInst &in, void (WasmJit::Function::*op)()) {
		BeginSet(in.dest);
		GetReg(in.src1);
		GetReg(in.src2);
		(f_.*op)();
		EndSet(in.dest);
	}
	enum class Fold { None, Add, Sub, And, Or, Xor, Shl, Shr, Sar };
	void BinConst(const IRInst &in, u32 src, u32 c, void (WasmJit::Function::*op)(), Fold fold) {
		u32 k = 0;
		const bool known = fold != Fold::None && KnownValue(src, k);
		BeginSet(in.dest);
		GetReg(src);
		f_.i32_const((s32)c);
		(f_.*op)();
		EndSet(in.dest);
		if (!known)
			return;
		u32 v = 0;
		switch (fold) {
		case Fold::Add: v = k + c; break;
		case Fold::Sub: v = k - c; break;
		case Fold::And: v = k & c; break;
		case Fold::Or: v = k | c; break;
		case Fold::Xor: v = k ^ c; break;
		case Fold::Shl: v = k << (c & 31); break;
		case Fold::Shr: v = k >> (c & 31); break;
		case Fold::Sar: v = (u32)((s32)k >> (c & 31)); break;
		case Fold::None: break;
		}
		Known(in.dest, true, v);
	}
	// Pushes the condition of a conditional exit (or its Else form) on src1, src2.
	void Cond(IROp op, const IRInst &in) {
		switch (op) {
		case IROp::ExitToConstIfEq: case IROp::OptExitToConstIfEqElse:
			GetReg(in.src1); GetReg(in.src2); f_.i32_eq(); break;
		case IROp::ExitToConstIfNeq: case IROp::OptExitToConstIfNeqElse:
			GetReg(in.src1); GetReg(in.src2); f_.i32_ne(); break;
		case IROp::ExitToConstIfGtZ: case IROp::OptExitToConstIfGtZElse:
			GetReg(in.src1); f_.i32_const(0); f_.i32_gt_s(); break;
		case IROp::ExitToConstIfGeZ: case IROp::OptExitToConstIfGeZElse:
			GetReg(in.src1); f_.i32_const(0); f_.i32_ge_s(); break;
		case IROp::ExitToConstIfLtZ: case IROp::OptExitToConstIfLtZElse:
			GetReg(in.src1); f_.i32_const(0); f_.i32_lt_s(); break;
		default:  // LeZ
			GetReg(in.src1); f_.i32_const(0); f_.i32_le_s(); break;
		}
	}
	void Mul64(const IRInst &in, bool sign) {
		GetReg(in.src1);
		if (sign) f_.i64_extend_i32_s(); else f_.i64_extend_i32_u();
		GetReg(in.src2);
		if (sign) f_.i64_extend_i32_s(); else f_.i64_extend_i32_u();
		f_.i64_mul();
	}
	// lo and hi from the i64 in ScratchI64, low word to lo: the interpreter's memcpy of lo and hi as one.
	void SplitLoHi() {
		const u32 t = ScratchI64();
		BeginSet(IRREG_LO); f_.local_get(t); f_.i32_wrap_i64(); EndSet(IRREG_LO);
		BeginSet(IRREG_HI); f_.local_get(t); f_.i64_const(32); f_.i64_shr_u(); f_.i32_wrap_i64(); EndSet(IRREG_HI);
	}
	void CallHelper(u32 fn, u32 type) {
		f_.i32_const((s32)fn);
		f_.call_indirect(type, 0);
	}
	void Count(int which) {
		if (!ctx_.countLinks)
			return;
		const u32 at = Cell(&g_linkCount[which]);
		f_.i32_const(0);
		f_.i32_const(0);
		f_.i32_load(at);
		f_.i32_const(1);
		f_.i32_add();
		f_.i32_store(at);
	}

	void EmitIsland(const IRInst *copy, IROp first) {
		BeforeCallOut();
		const u32 counter = (u32)(uintptr_t)&g_islandExec[(int)first];
		f_.i32_const((s32)counter);
		f_.i32_const((s32)counter);
		f_.i32_load(0);
		f_.i32_const(1);
		f_.i32_add();
		f_.i32_store(0);
		f_.local_get(0);
		f_.i32_const((s32)(uintptr_t)copy);
		CallHelper((u32)(uintptr_t)&pa_jit_island, types_.island);
		const u32 t = Scratch(0);
		f_.local_tee(t);
		f_.i32_const(1);
		f_.i32_ne();
		f_.if_();
		f_.local_get(t);
		f_.return_();
		f_.end();
		AfterCallOut();
	}

	// --- exits and links ---
	// Whether an exit to target can chain at all, by IR_EXIT's address test (a constant: decided here).
	bool Linkable(u32 target) const { return ctx_.links && ctx_.space && ctx_.space->IsValid4AlignedAddress(target); }
	void Return(u32 target) {
		f_.i32_const((s32)target);
		f_.return_();
	}
	void StorePc(u32 target) {
		f_.local_get(0);
		f_.i32_const((s32)target);
		f_.i32_store(OFF_PC);
	}
	// Pushes whether the block shadow at target still holds the block at entry.
	void ShadowHolds(u32 target, int entry) {
		f_.i32_const(0);
		f_.i32_load(ctx_.shadowBase + (target & kShadowMask));
		f_.i32_const(entry);
		f_.i32_eq();
	}
	// Pushes whether no ClearCache (nor a new JIT) came since this was compiled: arena offsets not reused.
	void EpochSame() {
		f_.i32_const(0);
		f_.i32_load(Cell(&g_epochCell));
		f_.i32_const((s32)g_epochCell);
		f_.i32_eq();
	}
	enum class Callee { Direct, Slot, SlotLocal };
	// Goes on in another block's code (the caller has stored mips->pc and every register): a tail call; without
	// tail calls a plain call, at most kMaxLinkDepth deep, whose result is returned. Falls through to the
	// caller's own return when the depth is spent. The counter (pa_jit_config 17) counts only a link taken.
	void Link(Callee kind, u32 which, int counter) {
		auto call = [&](bool tail) {
			f_.local_get(0);
			if (kind == Callee::Slot)
				f_.i32_const((s32)which);
			else if (kind == Callee::SlotLocal)
				f_.local_get(which);
			if (kind == Callee::Direct) {
				if (tail) f_.return_call(which); else f_.call(which);
			} else {
				if (tail) f_.return_call_indirect(types_.block, 0); else f_.call_indirect(types_.block, 0);
			}
		};
		if (ctx_.tailCalls) {
			Count(counter);
			call(true);
			return;
		}
		const u32 depth = Cell(&g_depthCell);
		f_.i32_const(0);
		f_.i32_load(depth);
		f_.i32_const((s32)kMaxLinkDepth);
		f_.i32_lt_u();
		f_.if_();
		Count(counter);
		f_.i32_const(0);
		f_.i32_const(0); f_.i32_load(depth); f_.i32_const(1); f_.i32_add();
		f_.i32_store(depth);
		call(false);
		f_.local_set(Scratch(10));
		f_.i32_const(0);
		f_.i32_const(0); f_.i32_load(depth); f_.i32_const(1); f_.i32_sub();
		f_.i32_store(depth);
		f_.local_get(Scratch(10));
		f_.return_();
		f_.end();
	}
	// Records a back edge's state in the first pass over a loop.
	void NoteBackEdge() {
		if (backEdges_++ == 0) {
			meet_ = s_;
		} else {
			meet_.loaded &= s_.loaded;
			meet_.dirty |= s_.dirty;
		}
	}
	// Brings the registers to the state the loop's top expects: what it holds in locals loaded, what it
	// counts clean stored.
	void Reconcile() {
		for (u32 r = 0; r < 256; ++r) {
			if (s_.dirty[r] && !top_.dirty[r])
				Flush(r);
		}
		for (u32 r = 0; r < 256; ++r) {
			if (top_.loaded[r] && !s_.loaded[r]) {
				LoadInto(r);
				s_.loaded[r] = true;
			}
		}
		if (s_.dcDirty && !top_.dcDirty) {
			f_.local_get(0);
			f_.local_get(DcLocal());
			f_.i32_store(OFF_DC);
			s_.dcDirty = false;
		}
	}
	// Leaves for a constant target on the path being emitted (a conditional exit saved the state around it).
	void ExitConst(u32 target) {
		ChargePending();
		if (loop_ && target == plan_->start && Linkable(target)) {
			// The block's own start: round again, with the registers kept in locals, while IR_EXIT would
			// chain here (the shadow holds this block whenever nothing ran that can invalidate it).
			DcNotNegative();
			if (mayInvalidate_) {
				ShadowHolds(target, plan_->entry);
				f_.i32_and();
				EpochSame();
				f_.i32_and();
			}
			f_.if_();
			{
				const RegState saved = s_;
				NoteBackEdge();
				Reconcile();
				StorePc(target);
				Count(LC_SELF);
				f_.br(f_.depth() - loopLabel_);
				s_ = saved;
			}
			f_.end();
			FlushAll();
			Return(target);
			return;
		}
		FlushAll();
		if (Linkable(target)) {
			const InModule *in = nullptr;
			if ((ctx_.links & LINK_DIRECT) && batch_) {
				auto it = batch_->find(target & kShadowMask);
				if (it != batch_->end())
					in = &it->second;
			}
			if (in) {
				DcNotNegative();
				ShadowHolds(target, in->entry);
				f_.i32_and();
				EpochSame();
				f_.i32_and();
				f_.if_();
				StorePc(target);
				Link(Callee::Direct, in->func, LC_DIRECT);
				f_.end();
			} else if ((ctx_.links & LINK_TABLE) && ctx_.linker.slotFor) {
				const int slot = ctx_.linker.SlotFor(target);
				if (slot > 0) {
					DcNotNegative();
					f_.if_();
					StorePc(target);
					Link(Callee::Slot, (u32)slot, LC_TABLE);
					f_.end();
				}
			}
		}
		Return(target);
	}
	// ExitToReg (fromReg) or ExitToPC: chains where IR_EXIT would, by the block shadow and the CallWasm head
	// it points at. Only main RAM's addresses ((v & 0x3E000003) == 0x08000000, the first case of
	// Memory::IsValid4AlignedAddress) are looked up here; any other goes back to IRInterpret, which tests
	// the rest. Offsets 0 and 1 hold Bad in a non-empty arena, which a running block's is.
	void ExitDynamic(bool fromReg, u32 reg) {
		const u32 v = Scratch(8);
		if (fromReg) {
			GetReg(reg);
		} else {
			f_.local_get(0);
			f_.i32_load(OFF_PC);
		}
		f_.local_set(v);
		ChargePending();
		FlushAll();
		if ((ctx_.links & LINK_REG) && ctx_.space) {
			DcNotNegative();
			f_.local_get(v); f_.i32_const(0x3E000003); f_.i32_and(); f_.i32_const(0x08000000); f_.i32_eq();
			f_.i32_and();
			f_.if_();
			{
				const u32 w = Scratch(9);
				f_.i32_const(0);
				f_.i32_load(Cell(&g_arenaCell));
				// The entry's address added as i32, wrapping as C's pointer sum does (a memarg offset would
				// not wrap: the fuzz's shadow base is a buffer minus the program's address).
				f_.local_get(v); f_.i32_const((s32)kShadowMask); f_.i32_and();
				f_.i32_const((s32)ctx_.shadowBase); f_.i32_add();
				f_.i32_load(0);
				f_.i32_const(3); f_.i32_shl();
				f_.i32_add();
				f_.i32_load(0);  // the head's op, dest, src1, src2: CallWasm and its slot
				f_.local_tee(w);
				f_.i32_const(0xFF); f_.i32_and(); f_.i32_const((s32)IROp::CallWasm); f_.i32_eq();
				f_.if_();
				f_.local_get(0); f_.local_get(v); f_.i32_store(OFF_PC);
				f_.local_get(w); f_.i32_const(8); f_.i32_shr_u(); f_.local_set(w);
				Link(Callee::SlotLocal, w, LC_REG);
				if (ctx_.countLinks) {
					// No CallWasm head (a found one whose link ran out of depth counts as neither).
					f_.else_();
					Count(LC_REGMISS);
				}
				f_.end();
			}
			f_.end();
		}
		f_.local_get(v);
		f_.return_();
	}
	// A conditional exit: what the taken path does to the state stays on it.
	void ExitIf(IROp op, const IRInst &in) {
		Cond(op, in);
		f_.if_();
		{
			const RegState saved = s_;
			const u32 pending = pendingDc_;
			ExitConst(in.constant);
			s_ = saved;
			pendingDc_ = pending;
		}
		f_.end();
	}
	void EmitElse(IROp op, const IRInst &in, u32 otherTarget) {
		if (!ctx_.links) {
			ChargePending();
			FlushAll();
			f_.i32_const((s32)in.constant);
			f_.i32_const((s32)otherTarget);
			Cond(op, in);
			f_.select();
			f_.return_();
			return;
		}
		ExitIf(op, in);
		ExitConst(otherTarget);
	}

	void EmitOp(IROp op, const IRInst &in) {
		using F = WasmJit::Function;
		switch (op) {
		case IROp::SetConst:
			BeginSet(in.dest); f_.i32_const((s32)in.constant); EndSet(in.dest);
			Known(in.dest, true, in.constant);
			break;
		case IROp::Mov: {
			u32 k = 0;
			const bool known = KnownValue(in.src1, k);
			BeginSet(in.dest); GetReg(in.src1); EndSet(in.dest);
			Known(in.dest, known, k);
			break;
		}
		case IROp::Add: Bin(in, &F::i32_add); break;
		case IROp::Sub: Bin(in, &F::i32_sub); break;
		case IROp::And: Bin(in, &F::i32_and); break;
		case IROp::Or: Bin(in, &F::i32_or); break;
		case IROp::Xor: Bin(in, &F::i32_xor); break;
		case IROp::AddConst: BinConst(in, in.src1, in.constant, &F::i32_add, Fold::Add); break;
		case IROp::SubConst: BinConst(in, in.src1, in.constant, &F::i32_sub, Fold::Sub); break;
		case IROp::AndConst: BinConst(in, in.src1, in.constant, &F::i32_and, Fold::And); break;
		case IROp::OrConst: BinConst(in, in.src1, in.constant, &F::i32_or, Fold::Or); break;
		case IROp::XorConst: BinConst(in, in.src1, in.constant, &F::i32_xor, Fold::Xor); break;
		// The Opt forms read dest, whatever src1 says.
		case IROp::OptAddConst: BinConst(in, in.dest, in.constant, &F::i32_add, Fold::Add); break;
		case IROp::OptAndConst: BinConst(in, in.dest, in.constant, &F::i32_and, Fold::And); break;
		case IROp::OptOrConst: BinConst(in, in.dest, in.constant, &F::i32_or, Fold::Or); break;
		case IROp::Neg: BeginSet(in.dest); f_.i32_const(0); GetReg(in.src1); f_.i32_sub(); EndSet(in.dest); break;
		case IROp::Not: BeginSet(in.dest); GetReg(in.src1); f_.i32_const(-1); f_.i32_xor(); EndSet(in.dest); break;
		// wasm masks shift counts by 31, as the interpreter's & 31 does.
		case IROp::Shl: Bin(in, &F::i32_shl); break;
		case IROp::Shr: Bin(in, &F::i32_shr_u); break;
		case IROp::Sar: Bin(in, &F::i32_shr_s); break;
		case IROp::Ror: Bin(in, &F::i32_rotr); break;
		case IROp::ShlImm: BinConst(in, in.src1, in.src2, &F::i32_shl, Fold::Shl); break;
		case IROp::ShrImm: BinConst(in, in.src1, in.src2, &F::i32_shr_u, Fold::Shr); break;
		case IROp::SarImm: BinConst(in, in.src1, in.src2, &F::i32_shr_s, Fold::Sar); break;
		case IROp::RorImm: BinConst(in, in.src1, in.src2, &F::i32_rotr, Fold::None); break;
		case IROp::Slt: Bin(in, &F::i32_lt_s); break;
		case IROp::SltU: Bin(in, &F::i32_lt_u); break;
		case IROp::SltConst: BinConst(in, in.src1, in.constant, &F::i32_lt_s, Fold::None); break;
		case IROp::SltUConst: BinConst(in, in.src1, in.constant, &F::i32_lt_u, Fold::None); break;
		case IROp::Clz: BeginSet(in.dest); GetReg(in.src1); f_.i32_clz(); EndSet(in.dest); break;
		// dest is also a source (IRFLAG_SRC3DST): select(src2, dest, cond).
		case IROp::MovZ: BeginSet(in.dest); GetReg(in.src2); GetReg(in.dest); GetReg(in.src1); f_.i32_eqz(); f_.select(); EndSet(in.dest); break;
		case IROp::MovNZ: BeginSet(in.dest); GetReg(in.src2); GetReg(in.dest); GetReg(in.src1); f_.select(); EndSet(in.dest); break;
		case IROp::Max:
			BeginSet(in.dest); GetReg(in.src1); GetReg(in.src2); GetReg(in.src1); GetReg(in.src2); f_.i32_gt_s(); f_.select(); EndSet(in.dest);
			break;
		case IROp::Min:
			BeginSet(in.dest); GetReg(in.src1); GetReg(in.src2); GetReg(in.src1); GetReg(in.src2); f_.i32_lt_s(); f_.select(); EndSet(in.dest);
			break;
		case IROp::BSwap16:
			BeginSet(in.dest);
			GetReg(in.src1); f_.i32_const((s32)0xFF00FF00); f_.i32_and(); f_.i32_const(8); f_.i32_shr_u();
			GetReg(in.src1); f_.i32_const(0x00FF00FF); f_.i32_and(); f_.i32_const(8); f_.i32_shl();
			f_.i32_or();
			EndSet(in.dest);
			break;
		case IROp::BSwap32:
			// rotr(x & 0x00FF00FF, 8) | rotl(x & 0xFF00FF00, 8)
			BeginSet(in.dest);
			GetReg(in.src1); f_.i32_const(0x00FF00FF); f_.i32_and(); f_.i32_const(8); f_.i32_rotr();
			GetReg(in.src1); f_.i32_const((s32)0xFF00FF00); f_.i32_and(); f_.i32_const(8); f_.i32_rotl();
			f_.i32_or();
			EndSet(in.dest);
			break;
		case IROp::Ext8to32: BeginSet(in.dest); GetReg(in.src1); f_.i32_extend8_s(); EndSet(in.dest); break;
		case IROp::Ext16to32: BeginSet(in.dest); GetReg(in.src1); f_.i32_extend16_s(); EndSet(in.dest); break;

		case IROp::MtLo: BeginSet(IRREG_LO); GetReg(in.src1); EndSet(IRREG_LO); break;
		case IROp::MtHi: BeginSet(IRREG_HI); GetReg(in.src1); EndSet(IRREG_HI); break;
		case IROp::MfLo: BeginSet(in.dest); GetReg(IRREG_LO); EndSet(in.dest); break;
		case IROp::MfHi: BeginSet(in.dest); GetReg(IRREG_HI); EndSet(in.dest); break;
		// lo and hi as one i64, as the interpreter's memcpy does, split into their locals.
		case IROp::Mult:
		case IROp::MultU:
			Mul64(in, op == IROp::Mult);
			f_.local_set(ScratchI64());
			SplitLoHi();
			break;
		case IROp::Madd:
		case IROp::MaddU:
		case IROp::Msub:
		case IROp::MsubU:
			GetReg(IRREG_LO); f_.i64_extend_i32_u();
			GetReg(IRREG_HI); f_.i64_extend_i32_u(); f_.i64_const(32); f_.i64_shl();
			f_.i64_or();
			Mul64(in, op == IROp::Madd || op == IROp::Msub);
			if (op == IROp::Madd || op == IROp::MaddU) f_.i64_add(); else f_.i64_sub();
			f_.local_set(ScratchI64());
			SplitLoHi();
			break;

		case IROp::Load8: { BeginSet(in.dest); const u32 o = Addr(in.src1, in.constant); f_.i32_load8_u(o); EndSet(in.dest); break; }
		case IROp::Load8Ext: { BeginSet(in.dest); const u32 o = Addr(in.src1, in.constant); f_.i32_load8_s(o); EndSet(in.dest); break; }
		case IROp::Load16: { BeginSet(in.dest); const u32 o = Addr(in.src1, in.constant); f_.i32_load16_u(o, 0); EndSet(in.dest); break; }
		case IROp::Load16Ext: { BeginSet(in.dest); const u32 o = Addr(in.src1, in.constant); f_.i32_load16_s(o, 0); EndSet(in.dest); break; }
		case IROp::Load32: { BeginSet(in.dest); const u32 o = Addr(in.src1, in.constant); f_.i32_load(o, 0); EndSet(in.dest); break; }
		case IROp::Load32Linked:
			if (in.dest != MIPS_REG_ZERO) {
				BeginSet(in.dest); const u32 o = Addr(in.src1, in.constant); f_.i32_load(o, 0); EndSet(in.dest);
			}
			f_.local_get(0);
			f_.i32_const(1);
			f_.i32_store(OFF_LLBIT);
			break;
		case IROp::LoadFloat: {
			f_.local_get(0); const u32 o = Addr(in.src1, in.constant); f_.i32_load(o, 0); StoreFI(in.dest);
			break;
		}
		case IROp::LoadVec4: {
			f_.local_get(0); const u32 o = Addr(in.src1, in.constant); f_.simd_mem(WasmJit::Simd::V128Load, o, 0); StoreV(in.dest);
			break;
		}
		case IROp::Store8: { const u32 o = Addr(in.src1, in.constant); GetReg(in.src3); f_.i32_store8(o); break; }
		case IROp::Store16: { const u32 o = Addr(in.src1, in.constant); GetReg(in.src3); f_.i32_store16(o, 0); break; }
		case IROp::Store32: { const u32 o = Addr(in.src1, in.constant); GetReg(in.src3); f_.i32_store(o, 0); break; }
		case IROp::Store32Conditional: {
			// dest and src3 are one field: the value stored, then where 1 or 0 goes.
			const u32 ll = Scratch(1), addr = Scratch(2), val = Scratch(3);
			f_.local_get(0); f_.i32_load(OFF_LLBIT); f_.local_set(ll);
			const u32 o = Addr(in.src1, in.constant); f_.local_set(addr);
			GetReg(in.src3); f_.local_set(val);
			f_.local_get(ll);
			f_.if_();
			f_.local_get(addr); f_.local_get(val); f_.i32_store(o, 0);
			f_.end();
			if (in.dest != MIPS_REG_ZERO) {
				BeginSet(in.dest); f_.local_get(ll); f_.i32_const(0); f_.i32_ne(); EndSet(in.dest);
			}
			break;
		}
		case IROp::StoreFloat: { const u32 o = Addr(in.src1, in.constant); LoadFI(in.src3); f_.i32_store(o, 0); break; }
		case IROp::StoreVec4: { const u32 o = Addr(in.src1, in.constant); LoadV(in.dest); f_.simd_mem(WasmJit::Simd::V128Store, o, 0); break; }
		case IROp::FMov: f_.local_get(0); LoadFI(in.src1); StoreFI(in.dest); break;
		case IROp::SetConstF: f_.local_get(0); f_.i32_const((s32)in.constant); StoreFI(in.dest); break;

		// --- M2: Div/DivU, with the branches wasm needs (its div traps on 0 and INT_MIN / -1) ---
		case IROp::Div:
		case IROp::DivU: {
			const bool sign = op == IROp::Div;
			const u32 n = Scratch(4), d = Scratch(5), lo = Scratch(6), hi = Scratch(7);
			GetReg(in.src1); f_.local_set(n);
			GetReg(in.src2); f_.local_set(d);
			f_.local_get(d);
			f_.i32_eqz();
			f_.if_();
			{
				// Div: lo = n < 0 ? 1 : -1. DivU: lo = n <= 0xFFFF ? 0xFFFF : -1. hi = n either way.
				f_.i32_const(sign ? 1 : 0xFFFF);
				f_.i32_const(-1);
				f_.local_get(n);
				if (sign) {
					f_.i32_const(0);
					f_.i32_lt_s();
				} else {
					f_.i32_const(0xFFFF);
					f_.i32_le_u();
				}
				f_.select();
				f_.local_set(lo);
				f_.local_get(n); f_.local_set(hi);
			}
			f_.else_();
			if (sign) {
				// The one overflow: lo = 0x80000000, hi = 0.
				f_.local_get(n); f_.i32_const((s32)0x80000000); f_.i32_eq();
				f_.local_get(d); f_.i32_const(-1); f_.i32_eq();
				f_.i32_and();
				f_.if_();
				f_.i32_const((s32)0x80000000); f_.local_set(lo);
				f_.i32_const(0); f_.local_set(hi);
				f_.else_();
			}
			f_.local_get(n); f_.local_get(d); if (sign) f_.i32_div_s(); else f_.i32_div_u(); f_.local_set(lo);
			f_.local_get(n); f_.local_get(d); if (sign) f_.i32_rem_s(); else f_.i32_rem_u(); f_.local_set(hi);
			if (sign)
				f_.end();
			f_.end();
			BeginSet(IRREG_LO); f_.local_get(lo); EndSet(IRREG_LO);
			BeginSet(IRREG_HI); f_.local_get(hi); EndSet(IRREG_HI);
			break;
		}

		// --- M2: the FPU ---
		case IROp::FAdd: FBin(in, &F::f32_add); break;
		case IROp::FSub: FBin(in, &F::f32_sub); break;
		case IROp::FDiv: FBin(in, &F::f32_div); break;
		case IROp::FMul: {
			// As the handler has it (ppsspp-3039c4e-wasmjit.patch): the product, unless it is a NaN (a NaN
			// operand, or inf * 0 either way round); then, from the operands' bits, the first NaN of (src1,
			// src2) with its quiet bit set, else 0x7fc00000. f32.mul's own NaN is never kept: for two NaNs
			// it is the engine's choice, and the hardware's rule looks at operand order (V8's x64 optimising
			// tier swapped the handler's operands and not this lowering's). The operands are read as i32,
			// so the NaN path is integer ops only.
			const u32 a = Scratch(4), b = Scratch(5), p = Scratch(6);
			LoadFI(in.src1); f_.local_tee(a); f_.f32_reinterpret_i32();
			LoadFI(in.src2); f_.local_tee(b); f_.f32_reinterpret_i32();
			f_.f32_mul(); f_.i32_reinterpret_f32(); f_.local_tee(p);
			IsNanBits();
			f_.if_unlikely();
			f_.local_get(a); f_.i32_const(0x00400000); f_.i32_or();
			f_.local_get(b); f_.i32_const(0x00400000); f_.i32_or();
			f_.i32_const(0x7fc00000);
			f_.local_get(b); IsNanBits();
			f_.select();
			f_.local_get(a); IsNanBits();
			f_.select();
			f_.local_set(p);
			f_.end();
			f_.local_get(0); f_.local_get(p);
			StoreFI(in.dest);
			break;
		}
		case IROp::FMin:
		case IROp::FMax: {
			const u32 a = Scratch(4), b = Scratch(5);
			LoadFI(in.src1); f_.local_set(a);
			LoadFI(in.src2); f_.local_set(b);
			f_.local_get(0);
			f_.local_get(a);
			f_.local_get(b);
			MinMaxKey(a);
			MinMaxKey(b);
			if (op == IROp::FMin) f_.i32_lt_s(); else f_.i32_gt_s();
			f_.select();
			StoreFI(in.dest);
			break;
		}
		case IROp::FSqrt:
			// A negative input gives 0x7FC00000.
			f_.local_get(0);
			f_.i32_const(0x7FC00000);
			LoadF(in.src1); f_.f32_sqrt(); f_.i32_reinterpret_f32();
			LoadF(in.src1); f_.f32_const(0.0f); f_.f32_lt();
			f_.select();
			StoreFI(in.dest);
			break;
		case IROp::FNeg: f_.local_get(0); LoadF(in.src1); f_.f32_neg(); StoreF(in.dest); break;
		case IROp::FAbs: f_.local_get(0); LoadF(in.src1); f_.f32_abs(); StoreF(in.dest); break;
		case IROp::FSign: {
			// Exponent 0 (zeros, denormals): 0.0f; else 1.0f or -1.0f by the sign bit.
			const u32 v = Scratch(4);
			LoadFI(in.src1); f_.local_set(v);
			f_.local_get(0);
			f_.i32_const(0);
			f_.i32_const(0x3F800000); f_.i32_const((s32)0xBF800000); f_.local_get(v); f_.i32_const(0); f_.i32_ge_s(); f_.select();
			f_.local_get(v); f_.i32_const(0x7F800000); f_.i32_and(); f_.i32_eqz();
			f_.select();
			StoreFI(in.dest);
			break;
		}
		case IROp::FSat0_1:
		case IROp::FSatMinus1_1: {
			// vfpu_clamp(v, min, 1): v >= 1 ? 1 : (v <= min ? min : v); a NaN stays, -0 becomes min = +0.
			const bool zero = op == IROp::FSat0_1;
			f_.local_get(0);
			f_.i32_const(0x3F800000);
			f_.i32_const(zero ? 0 : (s32)0xBF800000);
			LoadFI(in.src1);
			LoadF(in.src1); f_.f32_const(zero ? 0.0f : -1.0f); f_.f32_le();
			f_.select();
			LoadF(in.src1); f_.f32_const(1.0f); f_.f32_ge();
			f_.select();
			StoreFI(in.dest);
			break;
		}
		case IROp::FCvtSW: f_.local_get(0); LoadFI(in.src1); f_.f32_convert_i32_s(); StoreF(in.dest); break;
		case IROp::FCvtScaledSW: {
			// (float)x * (1.0f / (1 << n)): the scale is 2^-n exactly.
			const u32 bits = (127u - (in.src2 & 0x1F)) << 23;
			float scale;
			memcpy(&scale, &bits, 4);
			f_.local_get(0); LoadFI(in.src1); f_.f32_convert_i32_s(); f_.f32_const(scale); f_.f32_mul(); StoreF(in.dest);
			break;
		}
		case IROp::FRound: CallFpr((void *)&pa_jit_fround, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FTrunc: CallFpr((void *)&pa_jit_ftrunc, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FCeil: CallFpr((void *)&pa_jit_fceil, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FFloor: CallFpr((void *)&pa_jit_ffloor, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FCvtWS: CallFpr((void *)&pa_jit_fcvtws, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FCvtScaledWS: CallFpr((void *)&pa_jit_fcvtscaledws, types_.v4, {in.dest, in.src1, in.src2}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FMovFromGPR: f_.local_get(0); GetReg(in.src1); StoreFI(in.dest); break;
		case IROp::OptFCvtSWFromGPR: f_.local_get(0); GetReg(in.src1); f_.f32_convert_i32_s(); StoreF(in.dest); break;
		case IROp::FMovToGPR: BeginSet(in.dest); LoadFI(in.src1); EndSet(in.dest); break;
		case IROp::OptFMovToGPRShr8: BeginSet(in.dest); LoadFI(in.src1); f_.i32_const(8); f_.i32_shr_u(); EndSet(in.dest); break;

		// --- M2: fpcond, fcr31 and the VFPU control words (GPR-class registers 244, 245 and 208 on) ---
		case IROp::FpCondFromReg: BeginSet(IRREG_FPCOND); GetReg(in.src1); EndSet(IRREG_FPCOND); break;
		case IROp::FpCondToReg: BeginSet(in.dest); GetReg(IRREG_FPCOND); EndSet(in.dest); break;
		case IROp::FpCtrlFromReg:
			BeginSet(IRREG_FCR31); GetReg(in.src1); f_.i32_const(0x0181FFFF); f_.i32_and(); EndSet(IRREG_FCR31);
			BeginSet(IRREG_FPCOND); GetReg(IRREG_FCR31); f_.i32_const(23); f_.i32_shr_u(); f_.i32_const(1); f_.i32_and(); EndSet(IRREG_FPCOND);
			break;
		case IROp::FpCtrlToReg:
			BeginSet(IRREG_FCR31);
			GetReg(IRREG_FCR31); f_.i32_const(~(1 << 23)); f_.i32_and();
			GetReg(IRREG_FPCOND); f_.i32_const(1); f_.i32_and(); f_.i32_const(23); f_.i32_shl();
			f_.i32_or();
			EndSet(IRREG_FCR31);
			BeginSet(in.dest); GetReg(IRREG_FCR31); EndSet(in.dest);
			break;
		case IROp::VfpuCtrlToReg: BeginSet(in.dest); GetReg(IRREG_VFPU_CTRL_BASE + in.src1); EndSet(in.dest); break;
		case IROp::SetCtrlVFPU:
			BeginSet(IRREG_VFPU_CTRL_BASE + in.dest); f_.i32_const((s32)in.constant); EndSet(IRREG_VFPU_CTRL_BASE + in.dest);
			break;
		case IROp::SetCtrlVFPUReg: BeginSet(IRREG_VFPU_CTRL_BASE + in.dest); GetReg(in.src1); EndSet(IRREG_VFPU_CTRL_BASE + in.dest); break;
		case IROp::SetCtrlVFPUFReg: BeginSet(IRREG_VFPU_CTRL_BASE + in.dest); LoadFI(in.src1); EndSet(IRREG_VFPU_CTRL_BASE + in.dest); break;

		// --- M2: compares ---
		case IROp::FCmp: {
			// dest is the mode; a mode past LessEqualUnordered leaves fpcond as it is, as the switch does.
			if (in.dest > IRFpCompareMode::LessEqualUnordered)
				break;
			BeginSet(IRREG_FPCOND);
			switch (in.dest) {
			case IRFpCompareMode::False: f_.i32_const(0); break;
			case IRFpCompareMode::EitherUnordered:
				// !(a > b || a < b || a == b): either is a NaN.
				LoadF(in.src1); LoadF(in.src1); f_.f32_ne();
				LoadF(in.src2); LoadF(in.src2); f_.f32_ne();
				f_.i32_or();
				break;
			case IRFpCompareMode::EqualOrdered: LoadF(in.src1); LoadF(in.src2); f_.f32_eq(); break;
			case IRFpCompareMode::EqualUnordered:
				LoadF(in.src1); LoadF(in.src2); f_.f32_eq();
				LoadFI(in.src1); IsNanBits(); f_.i32_or();
				LoadFI(in.src2); IsNanBits(); f_.i32_or();
				break;
			case IRFpCompareMode::LessEqualOrdered: LoadF(in.src1); LoadF(in.src2); f_.f32_le(); break;
			case IRFpCompareMode::LessEqualUnordered: LoadF(in.src1); LoadF(in.src2); f_.f32_gt(); f_.i32_eqz(); break;
			case IRFpCompareMode::LessOrdered: LoadF(in.src1); LoadF(in.src2); f_.f32_lt(); break;
			default:  // LessUnordered
				LoadF(in.src1); LoadF(in.src2); f_.f32_ge(); f_.i32_eqz(); break;
			}
			EndSet(IRREG_FPCOND);
			break;
		}
		case IROp::FCmpVfpuBit: {
			// Bit (dest >> 4) of VFPU_CTRL_CC becomes condition (dest & 15) of src1, src2.
			const int cond = in.dest & 0xF;
			const u32 bit = 1u << (in.dest >> 4);
			BeginSet(IRREG_VFPU_CC);
			GetReg(IRREG_VFPU_CC);
			if (cond == VC_TR) {
				f_.i32_const((s32)bit);
				f_.i32_or();
				EndSet(IRREG_VFPU_CC);
				break;
			}
			f_.i32_const((s32)~bit);
			f_.i32_and();
			if (cond == VC_FL) {
				EndSet(IRREG_VFPU_CC);
				break;
			}
			switch (cond) {
			case VC_EQ: LoadF(in.src1); LoadF(in.src2); f_.f32_eq(); break;
			case VC_NE: LoadF(in.src1); LoadF(in.src2); f_.f32_ne(); break;
			case VC_LT: LoadF(in.src1); LoadF(in.src2); f_.f32_lt(); break;
			case VC_LE: LoadF(in.src1); LoadF(in.src2); f_.f32_le(); break;
			case VC_GT: LoadF(in.src1); LoadF(in.src2); f_.f32_gt(); break;
			case VC_GE: LoadF(in.src1); LoadF(in.src2); f_.f32_ge(); break;
			case VC_EZ: LoadF(in.src1); f_.f32_const(0.0f); f_.f32_eq(); break;
			case VC_NZ: LoadF(in.src1); f_.f32_const(0.0f); f_.f32_ne(); break;
			case VC_EN: LoadFI(in.src1); IsNanBits(); break;
			case VC_NN: LoadFI(in.src1); IsNanBits(); f_.i32_eqz(); break;
			case VC_EI: LoadFI(in.src1); IsInfBits(); break;
			case VC_NI: LoadFI(in.src1); IsInfBits(); f_.i32_eqz(); break;
			case VC_ES: LoadFI(in.src1); IsNanOrInfBits(); break;
			default:  // VC_NS
				LoadFI(in.src1); IsNanOrInfBits(); f_.i32_eqz(); break;
			}
			if (in.dest >> 4) {
				f_.i32_const(in.dest >> 4);
				f_.i32_shl();
			}
			f_.i32_or();
			EndSet(IRREG_VFPU_CC);
			break;
		}
		case IROp::FCmpVfpuAggregate: {
			// Bit 4: any of the mask's bits set; bit 5: all of them.
			const u32 mask = in.dest, cc = Scratch(4);
			GetReg(IRREG_VFPU_CC); f_.local_set(cc);
			BeginSet(IRREG_VFPU_CC);
			f_.local_get(cc); f_.i32_const(~0x30); f_.i32_and();
			f_.i32_const(0x10); f_.i32_const(0); f_.local_get(cc); f_.i32_const((s32)mask); f_.i32_and(); f_.select();
			f_.i32_or();
			f_.i32_const(0x20); f_.i32_const(0); f_.local_get(cc); f_.i32_const((s32)mask); f_.i32_and(); f_.i32_const((s32)mask); f_.i32_eq(); f_.select();
			f_.i32_or();
			EndSet(IRREG_VFPU_CC);
			break;
		}
		case IROp::FCmovVfpuCC:
			// f[dest] = f[src1] when CC bit (src2 & 15) is (src2 >> 7). The write is conditional: what a local
			// holds dirty over dest goes to memory first, and nothing changes the state inside the if.
			FprRead(in.src1, 1);
			FprRead(in.dest, 1);
			GetReg(IRREG_VFPU_CC);
			f_.i32_const(in.src2 & 0xF); f_.i32_shr_u(); f_.i32_const(1); f_.i32_and();
			f_.i32_const(in.src2 >> 7); f_.i32_eq();
			f_.if_();
			f_.local_get(0); LoadFI(in.src1); f_.i32_store(FprOff(in.dest));
			f_.end();
			FprWritten(in.dest, 1);
			break;

		// --- M2: Vec4, on v128 as the interpreter's SSE2 path compiles (checked in IRInterpret's disassembly) ---
		case IROp::Vec4Init: {
			alignas(16) static const float kInit[8][4] = {
				{0.0f, 0.0f, 0.0f, 0.0f}, {1.0f, 1.0f, 1.0f, 1.0f}, {-1.0f, -1.0f, -1.0f, -1.0f}, {1.0f, 0.0f, 0.0f, 0.0f},
				{0.0f, 1.0f, 0.0f, 0.0f}, {0.0f, 0.0f, 1.0f, 0.0f}, {0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f, 0.0f},
			};
			f_.local_get(0);
			f_.v128_const(kInit[in.src1 & 7]);
			StoreV(in.dest);
			break;
		}
		case IROp::Vec4Shuffle: {
			u8 lanes[16];
			for (int i = 0; i < 16; ++i)
				lanes[i] = (u8)(4 * Vec4ShuffleLane(in.src2, i / 4) + (i & 3));
			f_.local_get(0);
			LoadV(in.src1);
			f_.shuffle1(lanes, ScratchV(0));
			StoreV(in.dest);
			break;
		}
		case IROp::Vec4Blend: {
			u8 lanes[16];
			for (int i = 0; i < 16; ++i)
				lanes[i] = (u8)(((in.constant >> (i / 4)) & 1 ? 16 : 0) + i);
			f_.local_get(0);
			LoadV(in.src1);
			LoadV(in.src2);
			f_.shuffle(lanes);
			StoreV(in.dest);
			break;
		}
		case IROp::Vec4Mov: f_.local_get(0); LoadV(in.src1); StoreV(in.dest); break;
		case IROp::Vec4Add: VBin(in, WasmJit::Simd::F32x4Add); break;
		case IROp::Vec4Sub: VBin(in, WasmJit::Simd::F32x4Sub); break;
		case IROp::Vec4Mul: VBin(in, WasmJit::Simd::F32x4Mul); break;
		case IROp::Vec4Div: VBin(in, WasmJit::Simd::F32x4Div); break;
		case IROp::Vec4Scale:
			f_.local_get(0);
			LoadV(in.src1);
			FprRead(in.src2, 1);
			f_.local_get(0); f_.simd_mem(WasmJit::Simd::V128Load32Splat, FprOff(in.src2), 2);  // _mm_set1_ps compiles to this
			Simd(WasmJit::Simd::F32x4Mul);
			StoreV(in.dest);
			break;
		case IROp::Vec4Neg: f_.local_get(0); LoadV(in.src1); ConstV(0x80000000); Simd(WasmJit::Simd::V128Xor); StoreV(in.dest); break;
		case IROp::Vec4Abs: f_.local_get(0); LoadV(in.src1); ConstV(0x7FFFFFFF); Simd(WasmJit::Simd::V128And); StoreV(in.dest); break;
		case IROp::Vec4Min:
		case IROp::Vec4Max: {
			const u32 a = ScratchV(0), b = ScratchV(1);
			LoadV(in.src1); f_.local_set(a);
			LoadV(in.src2); f_.local_set(b);
			f_.local_get(0);
			f_.local_get(a);
			f_.local_get(b);
			MinMaxKeyV(a);
			MinMaxKeyV(b);
			Simd(op == IROp::Vec4Min ? WasmJit::Simd::I32x4LtS : WasmJit::Simd::I32x4GtS);
			Simd(WasmJit::Simd::V128Bitselect);
			StoreV(in.dest);
			break;
		}
		case IROp::Vec4Slt:
		case IROp::Vec4Sge:
			f_.local_get(0);
			LoadV(in.src1);
			LoadV(in.src2);
			Simd(op == IROp::Vec4Slt ? WasmJit::Simd::F32x4Lt : WasmJit::Simd::F32x4Ge);
			ConstV(0x3F800000);
			Simd(WasmJit::Simd::V128And);
			StoreV(in.dest);
			break;
		case IROp::Vec4CvtScaledSW: {
			const u32 bits = (127u - (in.src2 & 0x1F)) << 23;
			f_.local_get(0);
			ConstV(bits);
			LoadV(in.src1);
			Simd(WasmJit::Simd::F32x4ConvertI32x4S);
			Simd(WasmJit::Simd::F32x4Mul);
			StoreV(in.dest);
			break;
		}

		// --- M2: the VFPU's packs and unpacks, integer work on the bits ---
		case IROp::Vec2Unpack16To31:
		case IROp::Vec2Unpack16To32: {
			// dest = src << 16, dest + 1 = src & 0xFFFF0000 (each >> 1 for To31); src read once, first.
			const bool shift = op == IROp::Vec2Unpack16To31;
			const u32 x = Scratch(4);
			LoadFI(in.src1); f_.local_set(x);
			f_.local_get(0); f_.local_get(x); f_.i32_const(16); f_.i32_shl();
			if (shift) { f_.i32_const(1); f_.i32_shr_u(); }
			f_.i32_store(FprOff(in.dest));
			f_.local_get(0); f_.local_get(x); f_.i32_const((s32)0xFFFF0000); f_.i32_and();
			if (shift) { f_.i32_const(1); f_.i32_shr_u(); }
			f_.i32_store(FprOff(in.dest + 1));
			FprWritten(in.dest, 2);
			break;
		}
		case IROp::Vec4Unpack8To32: {
			// Lane i = byte i of src << 24 (the SSE2 path's unpacks and shift): one shuffle with zeros.
			u8 lanes[16];
			for (int i = 0; i < 16; ++i)
				lanes[i] = (u8)((i & 3) == 3 ? i / 4 : 16);
			f_.local_get(0);
			LoadFI(in.src1); Simd(WasmJit::Simd::I32x4Splat);
			ConstV(0);
			f_.shuffle(lanes);
			StoreV(in.dest);
			break;
		}
		case IROp::Vec4DuplicateUpperBitsAndShift1: {
			// Each lane: v |= v >> 8; v |= v >> 16; v >> 1.
			const u32 v = ScratchV(0);
			f_.local_get(0);
			LoadV(in.src1); f_.local_tee(v);
			f_.local_get(v); f_.i32_const(8); Simd(WasmJit::Simd::I32x4ShrU);
			Simd(WasmJit::Simd::V128Or); f_.local_tee(v);
			f_.local_get(v); f_.i32_const(16); Simd(WasmJit::Simd::I32x4ShrU);
			Simd(WasmJit::Simd::V128Or);
			f_.i32_const(1); Simd(WasmJit::Simd::I32x4ShrU);
			StoreV(in.dest);
			break;
		}
		case IROp::Vec4Pack32To8:
		case IROp::Vec4Pack31To8: {
			// As the SSE2 path compiles: each lane >> 24 (>> 23 arithmetic for Pack31: negative lanes then
			// saturate to 0), narrowed to 16 bits signed, then to 8 unsigned; the low 32 bits to dest.
			const u32 v = ScratchV(0);
			f_.local_get(0);
			LoadV(in.src1);
			if (op == IROp::Vec4Pack32To8) {
				f_.i32_const(24); Simd(WasmJit::Simd::I32x4ShrU);
			} else {
				f_.i32_const(23); Simd(WasmJit::Simd::I32x4ShrS);
			}
			f_.local_tee(v); f_.local_get(v); Simd(WasmJit::Simd::I16x8NarrowI32x4S);
			f_.local_tee(v); f_.local_get(v); Simd(WasmJit::Simd::I8x16NarrowI16x8U);
			f_.v128_store32_lane(FprOff(in.dest), 0);
			FprWritten(in.dest, 1);
			break;
		}
		case IROp::Vec2Pack32To16:
			// dest = (src >> 16) | (src + 1 & 0xFFFF0000)
			f_.local_get(0);
			LoadFI(in.src1); f_.i32_const(16); f_.i32_shr_u();
			LoadFI(in.src1 + 1); f_.i32_const((s32)0xFFFF0000); f_.i32_and();
			f_.i32_or();
			StoreFI(in.dest);
			break;
		case IROp::Vec2Pack31To16: {
			// Negative lanes count as 0: ((s0 >> 15) & 0xFFFF) | ((s1 << 1) & 0xFFFF0000).
			const u32 s0 = Scratch(4), s1 = Scratch(5);
			for (int k = 0; k < 2; ++k) {
				const u32 s = k ? s1 : s0;
				LoadFI(in.src1 + k); f_.local_set(s);
				f_.i32_const(0); f_.local_get(s); f_.local_get(s); f_.i32_const(0); f_.i32_lt_s(); f_.select(); f_.local_set(s);
			}
			f_.local_get(0);
			f_.local_get(s0); f_.i32_const(15); f_.i32_shr_u(); f_.i32_const(0xFFFF); f_.i32_and();
			f_.local_get(s1); f_.i32_const(1); f_.i32_shl(); f_.i32_const((s32)0xFFFF0000); f_.i32_and();
			f_.i32_or();
			StoreFI(in.dest);
			break;
		}
		case IROp::ReverseBits: {
			// ReverseBits32 (BitSet.h): swap bits, pairs, nibbles, bytes, halves.
			static const u32 kMasks[4] = {0x55555555, 0x33333333, 0x0F0F0F0F, 0x00FF00FF};
			const u32 v = Scratch(4);
			GetReg(in.src1); f_.local_set(v);
			for (int k = 0; k < 4; ++k) {
				const s32 sh = 1 << k;
				f_.local_get(v); f_.i32_const(sh); f_.i32_shr_u(); f_.i32_const((s32)kMasks[k]); f_.i32_and();
				f_.local_get(v); f_.i32_const((s32)kMasks[k]); f_.i32_and(); f_.i32_const(sh); f_.i32_shl();
				f_.i32_or();
				f_.local_set(v);
			}
			BeginSet(in.dest);
			f_.local_get(v); f_.i32_const(16); f_.i32_rotl();
			EndSet(in.dest);
			break;
		}

		// --- M2: the VFPU's own functions, by call ---
		case IROp::Vec4VfpuDot: CallFpr((void *)&pa_jit_vdot, types_.v4, {in.dest, in.src1, in.src2}, {{in.src1, 4}, {in.src2, 4}}, in.dest, 1); break;
		case IROp::Vec4VfpuMultiDot:
			// The block is up to 16 FPRs from src1 (rows or columns of the terms used), the vector up to 4.
			CallFpr((void *)&pa_jit_vmultidot, types_.v5, {in.dest, in.src1, in.src2, in.constant}, {{in.src1, 16}, {in.src2, 4}}, in.dest, 4);
			break;
		case IROp::FSin: CallFpr((void *)&pa_jit_fsin, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FCos: CallFpr((void *)&pa_jit_fcos, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FRSqrt: CallFpr((void *)&pa_jit_frsqrt, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FRecip: CallFpr((void *)&pa_jit_frecip, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FAsin: CallFpr((void *)&pa_jit_fasin, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FVSqrt: CallFpr((void *)&pa_jit_fvsqrt, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FExp2: CallFpr((void *)&pa_jit_fexp2, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FLog2: CallFpr((void *)&pa_jit_flog2, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 1); break;
		case IROp::FSinCos: CallFpr((void *)&pa_jit_fsincos, types_.v3, {in.dest, in.src1}, {{in.src1, 1}}, in.dest, 2); break;
		case IROp::FFloatToHalf2: CallFpr((void *)&pa_jit_ffloattohalf2, types_.v4, {in.dest, in.src1, in.src2}, {{in.src1, 1}, {in.src2, 1}}, in.dest, 1); break;
		case IROp::FHalfToFloat: CallFpr((void *)&pa_jit_fhalftofloat, types_.v4, {in.dest, in.src1, in.src2}, {{in.src1, 1}}, in.dest, 1); break;
		// These read fcr31 (in memory) and set the host's mode: no register of the guest's.
		case IROp::ApplyRoundingMode: Call((void *)&pa_jit_apply_rounding, types_.v1, {}); break;
		case IROp::RestoreRoundingMode: Call((void *)&pa_jit_restore_rounding, types_.v1, {}); break;
		case IROp::UpdateRoundingMode: break;  // the interpreter does nothing with it either

		case IROp::Downcount: pendingDc_ += in.constant; break;
		case IROp::SetPC: f_.local_get(0); GetReg(in.src1); f_.i32_store(OFF_PC); break;
		case IROp::SetPCConst: f_.local_get(0); f_.i32_const((s32)in.constant); f_.i32_store(OFF_PC); break;

		case IROp::ExitToConst: ExitConst(in.constant); break;
		case IROp::ExitToReg: {
			u32 k = 0;
			if (KnownValue(in.src1, k))
				ExitConst(k);
			else
				ExitDynamic(true, in.src1);
			break;
		}
		case IROp::ExitToPC: ExitDynamic(false, 0); break;
		case IROp::ExitToConstIfEq: case IROp::ExitToConstIfNeq: case IROp::ExitToConstIfGtZ:
		case IROp::ExitToConstIfGeZ: case IROp::ExitToConstIfLtZ: case IROp::ExitToConstIfLeZ:
			// The taken path charges what is pending; the other keeps it pending.
			ExitIf(op, in);
			break;

		case IROp::Syscall:
			BeforeCallOut();
			f_.local_get(0); f_.i32_const((s32)in.constant);
			CallHelper((u32)(uintptr_t)&pa_jit_syscall, types_.v2);
			AfterCallOut();
			break;
		case IROp::SyscallUnresolved:
			BeforeCallOut();
			f_.local_get(0); f_.i32_const((s32)in.constant);
			CallHelper((u32)(uintptr_t)&pa_jit_syscall_unresolved, types_.v2);
			AfterCallOut();
			break;
		case IROp::Interpret:
			BeforeCallOut();
			f_.local_get(0); f_.i32_const((s32)in.constant);
			CallHelper((u32)(uintptr_t)&pa_jit_interpret, types_.v2);
			AfterCallOut();
			break;
		case IROp::CallReplacement:
			BeforeCallOut();
			f_.local_get(0); f_.i32_const((s32)in.constant); f_.i32_const(in.dest);
			CallHelper((u32)(uintptr_t)&pa_jit_replacement, types_.v3);
			AfterCallOut();
			break;
		default:
			// MakePlan only lets through what Classify calls native or helper.
			f_.unreachable();
			break;
		}
	}

	WasmJit::Function &f_;
	const CompileCtx &ctx_;
	const Types &types_;
	const BatchMap *batch_;
	const Plan *plan_ = nullptr;
	int local_[256];
	int dcLocal_ = -1;
	RegState s_;
	u32 pendingDc_ = 0;
	bool mayInvalidate_ = false;
	// The loop of a self-loop block.
	bool loop_ = false;
	u32 loopLabel_ = 0;
	RegState top_;
	int backEdges_ = 0;
	RegState meet_;
	int scratch_[11] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
	int scratchV_[2] = {-1, -1};
	int scratchI64_ = -1;
};

// One block's function. A self-loop is compiled twice: the first pass finds what its back edges hold in
// locals, which the second keeps there across iterations (any state at the top would be correct: each
// back edge loads and stores to match it).
void CompileBlock(WasmJit::Function &f, const Plan &p, const std::vector<const IRInst *> &runCopies, const CompileCtx &ctx,
	const Types &types, const BatchMap &batch) {
	const bool loop = (ctx.links & LINK_SELF) && ctx.space && p.start && ctx.space->IsValid4AlignedAddress(p.start) &&
		BlockCompiler::HasSelfExit(p);
	if (!loop) {
		BlockCompiler bc(f, ctx, types, &batch);
		bc.Emit(p, runCopies, nullptr);
		return;
	}
	RegState top;
	{
		WasmJit::Function trial(1);
		BlockCompiler first(trial, ctx, types, &batch);
		first.Emit(p, runCopies, &top);
		if (first.SawBackEdge())
			top = first.BackEdges();
	}
	top.known.reset();
	top.dirty &= top.loaded;
	top.dcLoaded = true;  // every back edge tests it
	top.dcDirty = true;   // and charged it
	BlockCompiler bc(f, ctx, types, &batch);
	bc.Emit(p, runCopies, &top);
}

// Emits the plans as one module, loads it and fills their slots. 1, or 0 (an error, g_stats.lastError
// says), -1 (RangeError: the engine's code memory), -2 (no table slots). On success moduleId names
// the record that owns the slots and the island copies.
int CompileModule(const std::vector<Plan> &plans, const CompileCtx &ctx, std::vector<int> &slots, int &moduleId, size_t *bytesOut) {
	size_t islandOps = 0;
	for (const Plan &p : plans)
		for (const auto &run : p.runs)
			islandOps += (size_t)run.second + 1;
	std::unique_ptr<IRInst[]> islands(islandOps ? new IRInst[islandOps] : nullptr);

	WasmJit::Module m;
	Types types;
	types.block = m.type({ValType::I32}, {ValType::I32});
	types.island = m.type({ValType::I32, ValType::I32}, {ValType::I32});
	types.v1 = m.type({ValType::I32}, {});
	types.v2 = m.type({ValType::I32, ValType::I32}, {});
	types.v3 = m.type({ValType::I32, ValType::I32, ValType::I32}, {});
	types.v4 = m.type({ValType::I32, ValType::I32, ValType::I32, ValType::I32}, {});
	types.v5 = m.type({ValType::I32, ValType::I32, ValType::I32, ValType::I32, ValType::I32}, {});
	m.import_memory_unbounded("e", "m", 0);
	m.import_table("e", "t", 0);

	// The module's blocks by shadow entry: function b is plan b (no imported functions).
	BatchMap batch;
	if (ctx.links & LINK_DIRECT) {
		for (size_t b = 0; b < plans.size(); ++b) {
			if (plans[b].start)
				batch[plans[b].start & kShadowMask] = InModule{(u32)b, plans[b].entry};
		}
	}

	size_t at = 0;
	std::vector<const IRInst *> runCopies;
	char name[16];
	for (size_t b = 0; b < plans.size(); ++b) {
		const Plan &p = plans[b];
		runCopies.clear();
		for (const auto &run : p.runs) {
			IRInst *copy = islands.get() + at;
			memcpy(copy, p.ir + run.first, sizeof(IRInst) * run.second);
			copy[run.second] = IRInst{IROp::ExitToConst, {0}, 0, 0, 1};
			runCopies.push_back(copy);
			at += (size_t)run.second + 1;
		}
		WasmJit::Function f(1);
		CompileBlock(f, p, runCopies, ctx, types, batch);
		const u32 index = m.add_function(types.block, f);
		snprintf(name, sizeof(name), "b%u", (unsigned)b);
		m.export_function(name, index);
	}
	const std::vector<u8> bytes = m.finish();
	if (bytesOut)
		*bytesOut = bytes.size();

	if (!g_rt.Alloc((int)plans.size(), slots))
		return -2;
	const int result = pa_jit_load(bytes.data(), bytes.size(), (int)plans.size(), slots.data());
	if (result != 1) {
		for (int s : slots)
			g_rt.freeSlots.push_back(s);
		slots.clear();
		if (result == 0) {
			char buf[256];
			pa_jit_error(buf, sizeof(buf));
			g_stats.lastError = buf;
		}
		return result;
	}
	moduleId = g_rt.nextModule++;
	ModuleRec &rec = g_rt.modules[moduleId];
	rec.slots = slots;
	rec.live = (int)slots.size();
	rec.islands = std::move(islands);
	for (int s : slots)
		g_rt.slotModule[s] = moduleId;
	return 1;
}

typedef u32 (*BlockFn)(MIPSState *);

// --- the session: what outlives one IRWasmJit ----------------------------------------------------------
// A state load that keeps no compiled code deletes the CPU's JIT and makes a new one (MIPSState::DoState
// calls Reset when keepJitBlocks_ is empty), so what holds for the whole session lives here: why
// compiling stopped (JSC's spent code memory does not come back, §16.3.6: a stop is for the session),
// and the canary that watches for that.
const char *g_stopReason = nullptr;

void Stop(const char *why) {
	if (!g_stopReason)
		g_stopReason = why;
}

bool Stopped() { return g_stopReason != nullptr; }

// The code-memory canary (§16.3.6). JSC out of executable memory does not throw: its new code just never
// leaves the slow tiers, 35-80x slower than tiered code (§16.2), which is slower than the interpreter.
// Each window of frames a fresh canary block is compiled, called as a hot block would be for two thirds
// of the window, then timed up to three times; a window whose fastest timing is 5x the fastest of the
// good windows so far is slow. After one slow window compiling waits; after two in a row it stops for the
// session, and what was compiled since the last good window goes back to the interpreter (Revert).
struct Canary {
	int slot = 0;           // this window's canary, 0 when none is compiled
	int frame = 0;          // frames into the window
	u32 seq = 0;            // makes each canary fresh code
	u64 modulesBefore = 0;  // g_stats.modules when this window's canary was compiled
	u64 certified = 0;      // modules compiled before this one were compiled while fresh code tiered up
	int strikes = 0;        // slow windows in a row
	int timed = 0;          // timings this window
	double best = 0;        // this window's fastest timing, ns a call
	int coarseSeen = 0;     // timings in a row whose clock stepped by over 2 ms
	bool coarse = false;    // then the clock can't time it (a privacy mode's 16-100 ms steps): no canary
	alignas(16) u8 state[sizeof(MIPSState)] = {};
};
Canary g_canary;

bool CanaryWaits() { return g_cfg.canary && !g_canary.coarse && g_canary.strikes > 0; }

void ReleaseCanary() {
	if (g_canary.slot)
		g_rt.Release(g_canary.slot);
	g_canary.slot = 0;
	g_canary.frame = 0;
}

bool CompileCanary() {
	// Sixty integer ops on a scratch state; one constant differs each time, so it is fresh code.
	std::vector<IRInst> ir;
	ir.push_back(IRInst{IROp::Downcount, {0}, 0, 0, 1});
	ir.push_back(IRInst{IROp::SetConst, {9}, 0, 0, (u32)++g_canary.seq});
	for (int i = 0; i < 20; ++i) {
		ir.push_back(IRInst{IROp::Add, {(u8)(1 + i % 8)}, (u8)(1 + (i + 1) % 8), 9, 0});
		ir.push_back(IRInst{IROp::XorConst, {(u8)(1 + (i + 3) % 8)}, (u8)(1 + i % 8), 0, 0x9E3779B9u * (u32)(i + 1)});
		ir.push_back(IRInst{IROp::ShlImm, {(u8)(1 + (i + 5) % 8)}, (u8)(1 + (i + 2) % 8), (u8)(i % 31), 0});
	}
	ir.push_back(IRInst{IROp::ExitToConst, {0}, 0, 0, 0x08804000});
	Plan p;
	if (!MakePlan(ir.data(), (int)ir.size(), true, p))
		return false;
	std::vector<Plan> plans{p};
	std::vector<int> slots;
	int moduleId = 0;
	CompileCtx ctx;
	if (CompileModule(plans, ctx, slots, moduleId, nullptr) != 1)
		return false;
	g_canary.slot = slots[0];
	return true;
}

// The canary's ns a call, timed from one step of the clock to a later one. WebKit's clock moves in 1 ms
// steps and a call takes ~10 ns, so timing short runs counts ticks, not time: M1 summed 1,000 calls a
// frame, and a window's ~3 ms read as 1 to 11 ticks (3.4 to 37 ns), a "5x slower" one tick away from a
// stop. Starting at a step and calling until the clock reads kSpanMs past it times the span itself,
// whatever the clock's step. -1 if the step was over 2 ms (the clock can't time ~1 ms), or if the clock
// did not move in 4M calls (a frozen one).
double TimeCanary() {
	constexpr double kSpanMs = 1.0, kCoarseMs = 2.0;
	constexpr int kBatch = 64;
	constexpr u64 kMaxCalls = 1u << 22;
	BlockFn fn = (BlockFn)(uintptr_t)g_canary.slot;
	MIPSState *scratch = reinterpret_cast<MIPSState *>(g_canary.state);
	const double begin = Now();
	double edge;
	u64 calls = 0;
	while ((edge = Now()) == begin) {
		for (int i = 0; i < kBatch; ++i)
			fn(scratch);
		if ((calls += kBatch) >= kMaxCalls)
			return -1;
	}
	if (edge - begin > kCoarseMs)
		return -1;
	calls = 0;
	double end;
	do {
		for (int i = 0; i < kBatch; ++i)
			fn(scratch);
		if ((calls += kBatch) >= kMaxCalls)
			return -1;
	} while ((end = Now()) - edge < kSpanMs);
	return (end - edge) * 1e6 / (double)calls;
}

// Once a frame while compiling is on; lastCoreMs is the frame's own time. True when this frame's window
// stopped compiling (the caller then reverts what is not certified).
bool CanaryTick(double lastCoreMs) {
	constexpr int kWarmCalls = 1000, kTimings = 3;
	constexpr double kSlackMs = 11;  // a timing costs up to ~2 ms: only in frames that leave room for it
	Canary &c = g_canary;
	if (c.coarse)
		return false;
	const int window = std::max(g_cfg.canaryWindow, 10);
	if (c.slot == 0) {
		if (!CompileCanary())
			return false;
		c.frame = 0;
		c.timed = 0;
		c.best = 0;
		c.modulesBefore = g_stats.modules;
	}
	c.frame++;
	const bool last = c.frame >= window;
	if (c.frame <= window * 2 / 3) {
		// The calls a hot block gets: JSC tiers fresh code up after ~10k of them, V8 after 100k+ (§16.2).
		BlockFn fn = (BlockFn)(uintptr_t)c.slot;
		MIPSState *scratch = reinterpret_cast<MIPSState *>(c.state);
		for (int i = 0; i < kWarmCalls; ++i)
			fn(scratch);
	} else if (c.timed < kTimings && (lastCoreMs <= kSlackMs || (last && c.timed == 0))) {
		// Up to three timings in frames with room (the last frame takes one regardless if none did); the
		// fastest counts, as being preempted only ever makes one slower.
		const double ns = TimeCanary();
		if (ns < 0) {
			if (++c.coarseSeen >= 3) {
				c.coarse = true;
				ReleaseCanary();
				return false;
			}
		} else {
			c.coarseSeen = 0;
			c.best = c.timed ? std::min(c.best, ns) : ns;
			c.timed++;
			g_stats.canaryTimings++;
		}
	}
	if (!last)
		return false;
	ReleaseCanary();
	g_stats.canaryWindows++;
	if (c.timed == 0)
		return false;
	g_stats.canaryLastNs = c.best;
	if (g_stats.canaryBaseNs > 0 && c.best >= 5 * g_stats.canaryBaseNs) {
		g_stats.canaryStrikes++;
		if (++c.strikes < 2)
			return false;
		Stop("canary");
		return true;
	}
	// A good window: what was compiled before its canary was compiled while fresh code still tiered up.
	g_stats.canaryBaseNs = g_stats.canaryBaseNs > 0 ? std::min(g_stats.canaryBaseNs, c.best) : c.best;
	c.strikes = 0;
	c.certified = c.modulesBefore;
	return false;
}

// --- the JIT -------------------------------------------------------------------------------------------
class IRWasmJit;
IRWasmJit *g_jit = nullptr;
bool g_wanted = false;

void SetHeat(IRInst &head, u32 heat) {
	head.dest = (u8)heat;
	head.src1 = (u8)(heat >> 8);
}

class IRWasmJit : public MIPSComp::IRInterpreterJit {
public:
	explicit IRWasmJit(MIPSState *mips) : IRInterpreterJit(mips) {
		g_jit = this;
		g_paWasmJitHot = &IRWasmJit::OnHot;
		g_stats.callWasmBase = g_paCallWasmCount;
		g_stats.blockEntryBase = g_paBlockEntryCount;
		g_stats.instances++;
		g_epochCell++;
		PublishArena();
	}
	~IRWasmJit() override {
		if (g_jit == this) {
			g_jit = nullptr;
			g_paWasmJitHot = nullptr;
		}
		g_epochCell++;
		// The table entries go with the module; nothing of ours runs any more. The canary is the
		// session's: the next JIT goes on with its window.
		for (const auto &entry : slotEntry_)
			g_rt.Release(entry.first);
		for (const auto &link : links_)
			g_rt.Release(link.second.slot);
		for (int s : pendingFree_)
			g_rt.Release(s);
	}

	static void OnHot(const IRInst *inst) {
		if (g_jit)
			g_jit->Hot(inst);
	}

	void ClearCache() override {
		// Every compiled block goes with the arena, and every link slot with them: their slots are freed at
		// the next Frame (ClearCache never runs while compiled code is on the stack: only the dispatcher,
		// RunLoopUntil's deferred work and state loads call it).
		for (const auto &entry : slotEntry_)
			pendingFree_.push_back(entry.first);
		slotEntry_.clear();
		for (const auto &link : links_)
			pendingFree_.push_back(link.second.slot);
		links_.clear();
		hot_.clear();
		rejected_.clear();
		g_stats.clears++;
		const IRInst *before = arena_.data();
		IRInterpreterJit::ClearCache();
		if (arena_.data() != before)
			g_paArenaGen++;
		g_epochCell++;
		PublishArena();
	}

	void Frame(double lastCoreMs) {
		g_stats.frames++;
		for (int s : pendingFree_)
			g_rt.Release(s);
		pendingFree_.clear();
		if (Stopped() || !g_cfg.canary)
			ReleaseCanary();
		if (Stopped())
			return;
		if (!g_rt.Init()) {
			Stop("no table");
			return;
		}
		CheckBakeKey();
		if (g_cfg.canary && CanaryTick(lastCoreMs)) {
			Revert(g_canary.certified);
			return;
		}
		// After one slow canary window compiling waits for the next one's verdict: if JSC's cliff has
		// come, whatever compiled now would run slower than the interpreter and only be reverted.
		if (Stopped() || hot_.empty() || CanaryWaits())
			return;

		const double start = Now();
		const double budgetMs = std::min(std::max(14.0 - lastCoreMs, 0.2), g_cfg.budgetMaxUs / 1000.0);
		std::vector<Plan> batch;
		for (int modules = 0; modules < g_cfg.maxModules && !hot_.empty() && !Stopped();) {
			BuildBatch(batch);
			if (batch.empty())
				continue;
			CompileBatch(batch);
			modules++;
			if (Now() - start >= budgetMs)
				break;
		}
		const double ms = Now() - start;
		g_stats.compileMs += ms;
		g_stats.compileMsMax = std::max(g_stats.compileMsMax, ms);
		g_stats.framesCompiling++;
	}

	size_t LiveBlocks() const { return slotEntry_.size(); }
	size_t HotQueued() const { return hot_.size(); }
	size_t LinkSlots() const { return links_.size(); }

	// After the canary stopped compiling: the blocks of modules compiled from fromModule on may be the
	// cliff's, code that never leaves JSC's slowest tier. They go back to the interpreter (outside
	// IRInterpret: Frame, or a test), their slots freed at the next Frame; nothing compiles again.
	// Freeing gives JSC no code memory back (§16.2), so the certified blocks stay. A link slot holding
	// one of them gets the stub again; direct links stay inside a module, which goes back whole.
	void Revert(u64 fromModule) {
		for (auto it = slotEntry_.begin(); it != slotEntry_.end();) {
			if (it->second.module < fromModule) {
				++it;
				continue;
			}
			IRInst &head = arena_[it->second.entry];
			if (head.op == IROp::CallWasm)
				head = IRInst{IROp::Downcount, {0}, 0, 0, head.constant};
			Unlink(it->second.start, it->second.entry);
			pendingFree_.push_back(it->first);
			it = slotEntry_.erase(it);
			g_stats.reverted++;
		}
	}

	// The link slot standing for constant target `target` (one per block shadow entry), made the first time
	// a block exits there: it holds the compiled code of the block the shadow holds there now, or the stub
	// (the Linker; called while compiling).
	static int SlotForThunk(void *self, u32 target) { return static_cast<IRWasmJit *>(self)->SlotFor(target); }
	int SlotFor(u32 target) {
		const u32 key = target & kShadowMask;
		auto it = links_.find(key);
		if (it != links_.end())
			return it->second.slot;
		if (g_rt.stubSlot <= 0)
			return 0;
		std::vector<int> slot;
		if (!g_rt.Alloc(1, slot))
			return 0;
		LinkSlot link{slot[0], 0};
		const u32 offset = *mips_->space->GetBlockShadowEntry(target);
		if (offset > 1 && offset < arena_.size() && arena_[offset].op == IROp::CallWasm)
			SetLink(link, (int)offset, SlotOf(arena_[offset]));
		else
			SetLink(link, 0, g_rt.stubSlot);
		links_[key] = link;
		return link.slot;
	}

	// Invalidates code as the guest's icache op would, from a syscall's wrapper while compiled code runs
	// (pa_jit_config 18): alternately the 64 bytes around the PC (usually the block making the call) and
	// the start of one of the compiled blocks.
	void Chaos(u32 pc) {
		g_stats.chaos++;
		if (g_stats.chaos & 1) {
			InvalidateCacheAt(pc & ~0x3Fu, 0x40);
			return;
		}
		if (slotEntry_.empty())
			return;
		auto it = slotEntry_.begin();
		std::advance(it, (size_t)((g_stats.chaos / 2) % slotEntry_.size()));
		const u32 start = it->second.start;
		InvalidateCacheAt(start, 4);
	}

	// --- tests only: the chained fuzz builds its program with these ---
	// A block of the given IR at addr, as CompileBlock adds one (its head a counting BlockEntry); its entry.
	int TestAddBlock(u32 addr, u32 bytes, const std::vector<IRInst> &ir) {
		const int num = blocks_.AllocateBlock((int)addr, bytes, (u32)ir.size());
		CompileTargetBlock(num, ir);
		blocks_.FinalizeBlock(num);
		FinalizeTargetBlock(num);
		return blocks_.GetBlock(num)->GetEntry();
	}
	// Compiles these blocks now, in batches of g_cfg.batch; how many compiled. Not through hot_: a second
	// user of its deque would change what -flto inlines into Hot(), and so into IRInterpret's BlockEntry.
	int TestCompile(const std::vector<int> &entries) {
		const u64 before = g_stats.blocks;
		std::vector<Plan> batch;
		std::unordered_set<int> inBatch;
		plainLoad16_ = Ctx().plainLoad16;
		for (size_t i = 0; i < entries.size() && !Stopped(); ++i) {
			arena_[entries[i]].op = IROp::Downcount;
			Candidate(entries[i], batch, inBatch, false);
			if ((int)batch.size() >= g_cfg.batch || i + 1 == entries.size()) {
				if (!batch.empty())
					CompileBatch(batch);
				batch.clear();
				inBatch.clear();
			}
		}
		return (int)(g_stats.blocks - before);
	}
	const IRInst *TestArena() const { return arena_.data(); }

protected:
	CompileResult CompileTargetBlock(int block_num, const std::vector<IRInst> &instructions) override {
		const IRInst *before = arena_.data();
		const CompileResult result = IRInterpreterJit::CompileTargetBlock(block_num, instructions);
		// This can run inside IRInterpret (IRJit.cpp's preload warning), even inside a compiled block's
		// helper: CallWasm then returns instead of chaining through the old arena.
		if (arena_.data() != before) {
			g_paArenaGen++;
			PublishArena();
		}
		if (result == CompileResult::OK && !Stopped()) {
			IRInst &head = arena_[blocks_.GetBlock(block_num)->GetEntry()];
			if (head.op == IROp::Downcount) {
				head.op = IROp::BlockEntry;
				SetHeat(head, (u32)std::max(1, std::min(g_cfg.thresh, 0xFFFF)));
			}
		}
		return result;
	}

	// A new block took over its shadow entry (FinalizeBlock wrote it): a link slot there holding another
	// block's code (one at a mirror of the address) gets the stub; the new block is not compiled yet.
	void FinalizeTargetBlock(int block_num) override {
		IRInterpreterJit::FinalizeTargetBlock(block_num);
		if (links_.empty())
			return;
		const MIPSComp::JitBlock *block = blocks_.GetBlock(block_num);
		auto it = links_.find(block->GetOriginalStart() & kShadowMask);
		if (it != links_.end() && it->second.owner != 0 && it->second.owner != block->GetEntry())
			SetLink(it->second, 0, g_rt.stubSlot);
	}

	void InvalidateTargetBlock(int block_num) override {
		const MIPSComp::JitBlock *block = blocks_.GetBlock(block_num);
		const int entry = block ? block->GetEntry() : 0;
		if (entry <= 1 || entry >= (int)arena_.size())
			return;
		// Before Destroy clears the shadow: no link may reach this block's code from now on. Its own
		// shadow check stops a loop or a direct link; its link slot gets the stub here.
		Unlink(block->GetOriginalStart(), entry);
		IRInst &head = arena_[entry];
		if (head.op == IROp::CallWasm) {
			// A compiled block that is running now runs on to its exit; its slot goes at the next Frame.
			const int slot = SlotOf(head);
			pendingFree_.push_back(slot);
			slotEntry_.erase(slot);
			head = IRInst{IROp::Downcount, {0}, 0, 0, head.constant};
			g_stats.invalidated++;
		}
		// A BlockEntry, or a hot block waiting: the compile looks again (the block is gone by then).
	}

	// The debugger's copy of a block: its head as the interpreter wrote it.
	void GetBlockInstructions(const MIPSComp::JitBlock &block, std::vector<IRInst> &instructions) const override {
		IRInterpreterJit::GetBlockInstructions(block, instructions);
		if (!instructions.empty() && (instructions[0].op == IROp::BlockEntry || instructions[0].op == IROp::CallWasm))
			instructions[0] = IRInst{IROp::Downcount, {0}, 0, 0, instructions[0].constant};
	}

private:
	struct LinkSlot {
		int slot;   // its table slot, which links call through
		int owner;  // the arena offset of the block whose code it holds, 0 for the stub
	};

	static int SlotOf(const IRInst &callWasm) { return callWasm.dest | (callWasm.src1 << 8) | (callWasm.src2 << 16); }

	void SetLink(LinkSlot &link, int owner, int fromSlot) {
		pa_jit_table_copy(link.slot, fromSlot);
		link.owner = owner;
	}
	// The block at entry (start its address) is no longer compiled, or no longer the shadow's there.
	void Unlink(u32 start, int entry) {
		if (links_.empty())
			return;
		auto it = links_.find(start & kShadowMask);
		if (it != links_.end() && it->second.owner == entry)
			SetLink(it->second, 0, g_rt.stubSlot);
	}

	// ExitToReg's lookups read the arena's address from this cell.
	void PublishArena() {
		if (g_jit == this)
			g_arenaCell = (u32)(uintptr_t)arena_.data();
	}

	void Hot(const IRInst *inst) {
		const int offset = (int)(inst - arena_.data());
		IRInst &head = arena_[offset];
		auto rej = rejected_.find(offset);
		if (rej != rejected_.end()) {
			// A block that would not compile counts its entries in long rounds, for the stats.
			g_stats.rejectedBy[(int)rej->second].exec += 0xFFFF;
			head.op = IROp::BlockEntry;
			SetHeat(head, 0xFFFF);
			return;
		}
		if (Stopped())
			return;  // stays a plain Downcount
		if (hot_.size() >= 1024) {
			head.op = IROp::BlockEntry;
			SetHeat(head, (u32)std::max(1, g_cfg.thresh / 2));
			return;
		}
		hot_.push_back(offset);
	}

	// The addresses compiled code holds as constants: if one moved, all of it goes (PlayAgain boots one
	// game a module, so this should never happen).
	void CheckBakeKey() {
		const u8 *base = mips_->space->base;
		const u8 *shadow = mips_->space->blockShadow;
		if (baked_ && (base != bakedBase_ || shadow != bakedShadow_)) {
			for (const auto &entry : slotEntry_) {
				IRInst &head = arena_[entry.second.entry];
				if (head.op == IROp::CallWasm)
					head = IRInst{IROp::Downcount, {0}, 0, 0, head.constant};
				pendingFree_.push_back(entry.first);
			}
			slotEntry_.clear();
			for (auto &link : links_)
				SetLink(link.second, 0, g_rt.stubSlot);
			g_stats.flushes++;
		}
		baked_ = true;
		bakedBase_ = base;
		bakedShadow_ = shadow;
	}

	CompileCtx Ctx() {
		CompileCtx ctx;
		ctx.memBase = (u32)(uintptr_t)mips_->space->base;
		ctx.plainLoad16 = !Memory::DepthMirrorsActive() || mips_->space->IsMediaEngine();
		ctx.writeBack = g_cfg.writeBack != 0;
		ctx.foldConst = g_cfg.foldConst != 0;
		ctx.space = mips_->space;
		ctx.shadowBase = (u32)(uintptr_t)mips_->space->blockShadow;
		ctx.links = ctx.shadowBase ? (u32)g_cfg.links : 0;
		ctx.tailCalls = g_rt.tailCalls && g_cfg.tailCalls;
		ctx.countLinks = g_cfg.countLinks != 0;
		ctx.linker.slotFor = &IRWasmJit::SlotForThunk;
		ctx.linker.self = this;
		return ctx;
	}

	// Checks a block is still what its arena offset says, and plans it. Rejected blocks keep counting.
	bool Candidate(int offset, std::vector<Plan> &batch, std::unordered_set<int> &inBatch, bool neighbour) {
		if (inBatch.count(offset) || offset <= 1 || offset >= (int)arena_.size())
			return false;
		IRInst &head = arena_[offset];
		if (neighbour) {
			if (head.op == IROp::BlockEntry) {
				const u32 left = head.dest | (head.src1 << 8);
				const u32 thresh = (u32)g_cfg.thresh;
				// A neighbour that ran at least a quarter of the threshold comes along.
				if (rejected_.count(offset) || left > thresh || thresh - left < thresh / 4)
					return false;
			} else if (head.op != IROp::Downcount) {
				return false;
			}
		} else if (head.op != IROp::Downcount) {
			return false;
		}
		const int num = blocks_.FindByEntry(offset);
		if (num < 0)
			return false;
		const MIPSComp::JitBlock *block = blocks_.GetBlock(num);
		if (!block->IsValid() || *mips_->space->GetBlockShadowEntry(block->GetOriginalStart()) != (u32)offset)
			return false;
		if ((int)g_stats.blocks + (int)batch.size() >= g_cfg.maxBlocks) {
			Stop("block cap");
			return false;
		}
		Plan p;
		const bool ok = MakePlan(&arena_[offset], block->GetNumIRInstructions(), plainLoad16_, p);
		if (!ok || (g_cfg.islandFactor > 0 && p.nativeOps < g_cfg.islandFactor * (int)p.runs.size())) {
			Reject(offset, p, ok);
			return false;
		}
		p.entry = offset;
		p.start = block->GetOriginalStart();
		inBatch.insert(offset);
		batch.push_back(std::move(p));
		return true;
	}

	void Reject(int offset, const Plan &p, bool planned) {
		g_stats.rejected++;
		IROp why = p.rejectOp;
		if (!planned && p.noExit) {
			g_stats.rejectedNoExit++;
		} else if (planned) {
			// Too many islands: blame the op that starts the most of them.
			int count[256] = {};
			int best = -1;
			for (const auto &run : p.runs) {
				const int op = (int)p.ir[run.first].op;
				count[op]++;
				if (best < 0 || count[op] > count[best])
					best = op;
			}
			if (best >= 0)
				why = (IROp)best;
		}
		g_stats.rejectedBy[(int)why].blocks++;
		g_stats.rejectedBy[(int)why].exec += (u64)g_cfg.thresh;
		rejected_[offset] = why;
		IRInst &head = arena_[offset];
		head.op = IROp::BlockEntry;
		SetHeat(head, 0xFFFF);
	}

	void BuildBatch(std::vector<Plan> &batch) {
		batch.clear();
		std::unordered_set<int> inBatch;
		plainLoad16_ = Ctx().plainLoad16;
		while (!hot_.empty() && (int)batch.size() < g_cfg.batch && !Stopped()) {
			const int offset = hot_.front();
			hot_.pop_front();
			if (!Candidate(offset, batch, inBatch, false))
				continue;
			// Blocks of the same 4 KB guest page that have run some come along.
			const MIPSComp::JitBlock *block = blocks_.GetBlock(blocks_.FindByEntry(offset));
			const u32 page = block->GetOriginalStart() & ~0xFFFu;
			for (int num : blocks_.FindInvalidatedBlockNumbers(page, 0x1000)) {
				if ((int)batch.size() >= g_cfg.batch || Stopped())
					break;
				const MIPSComp::JitBlock *other = blocks_.GetBlock(num);
				if (other && other->IsValid())
					Candidate(other->GetEntry(), batch, inBatch, true);
			}
		}
	}

	void CompileBatch(std::vector<Plan> &batch) {
		std::vector<int> slots;
		int moduleId = 0;
		size_t bytes = 0;
		const int result = CompileModule(batch, Ctx(), slots, moduleId, &bytes);
		if (result == 1) {
			const u64 module = g_stats.modules++;  // the canary certifies blocks by it
			g_stats.blocks += batch.size();
			g_stats.bytes += bytes;
			for (size_t i = 0; i < batch.size(); ++i) {
				const Plan &p = batch[i];
				const int slot = slots[i];
				IRInst &head = arena_[p.entry];
				head = IRInst{IROp::CallWasm, {(u8)slot}, (u8)(slot >> 8), (u8)(slot >> 16), p.dc0};
				slotEntry_[slot] = Live{p.entry, module, p.start};
				// Links to this block's address now reach its code.
				auto link = links_.find(p.start & kShadowMask);
				if (link != links_.end())
					SetLink(link->second, p.entry, slot);
				for (const auto &run : p.runs)
					g_stats.islands[(int)p.ir[run.first].op].blocks++;
			}
			return;
		}
		if (result == -1) {
			// The engine is out of code memory: what is compiled stays, nothing more is (§16.3.6).
			g_stats.rangeErrors++;
			Stop("RangeError");
		} else if (result == -2) {
			Stop("no table slots");
		} else {
			g_stats.loadErrors++;
			fprintf(stderr, "pa_wasmjit: a module of %d blocks did not load: %s\n", (int)batch.size(), g_stats.lastError.c_str());
			for (const Plan &p : batch) {
				Plan failed = p;
				failed.rejectOp = IROp::Bad;
				Reject(p.entry, failed, false);
			}
		}
	}

	struct Live {
		int entry;   // arena offset of the block's head
		u64 module;  // g_stats.modules before its module was compiled: what the canary certifies by
		u32 start;   // its guest address
	};
	std::deque<int> hot_;
	std::unordered_map<int, Live> slotEntry_;     // live slot -> its block
	std::unordered_map<int, IROp> rejected_;      // arena offset -> the op blamed
	std::unordered_map<u32, LinkSlot> links_;     // block shadow entry -> its link slot
	std::vector<int> pendingFree_;
	bool baked_ = false;
	const u8 *bakedBase_ = nullptr;
	const u8 *bakedShadow_ = nullptr;
	bool plainLoad16_ = true;
};

// Every so many syscalls from compiled code (pa_jit_config 18; tests only).
u32 g_chaosCalls = 0;

MIPSComp::IRInterpreterJit *CreateJit(MIPSState *mips) {
	if (g_wanted)
		return new IRWasmJit(mips);
	return new MIPSComp::IRInterpreterJit(mips);
}

// --- L1: the fuzz --------------------------------------------------------------------------------------
struct Rng {
	u32 s;
	u32 Next() {
		s ^= s << 13;
		s ^= s >> 17;
		s ^= s << 5;
		return s;
	}
	u32 Below(u32 n) { return Next() % n; }
	bool Chance(u32 pct) { return Below(100) < pct; }
};

const u32 kEdge[] = {0, 1, 2, 0xFFFFFFFF, 0x80000000, 0x7FFFFFFF, 0xFFFF, 0x8000, 0x7FFF, 0xFF, 0x80, 0x7F,
	31, 32, 33, 0xFFFFFF80, 0x12345678, 0xFFFF0000, 0x0000FFFF, 0xFFFFFFFE, 0x80000001};
const u32 kFloatEdge[] = {
	// Zeros, ones, infinities.
	0, 0x80000000, 0x3F800000, 0xBF800000, 0x7F800000, 0xFF800000,
	// NaNs: quiet and signalling, both signs, payloads high and low.
	0x7FC00000, 0xFFC00000, 0x7F800001, 0xFF800001, 0x7FC00001, 0x7FFFFFFF, 0xFFFFFFFF, 0x7FBFFFFF, 0xFFA5A5A5, 0x7FD00F00,
	// Denormals and the smallest normals.
	0x00000001, 0x80000001, 0x807FFFFF, 0x007FFFFF, 0x00400000, 0x00800000, 0x80800000,
	// The largest finite, around the int32 limits, and 2^23, 2^24.
	0x7F7FFFFF, 0xFF7FFFFF, 0x4F000000, 0xCF000000, 0x4EFFFFFF, 0xCEFFFFFF, 0xCF000001, 0x4B000000, 0x4B800000,
	// Halves and near-halves (rounding), and small integers.
	0x3F000000, 0xBF000000, 0x3FC00000, 0xBFC00000, 0x40200000, 0xC0200000, 0x3EFFFFFF, 0xBEFFFFFF, 0x3F000001,
	0x4AFFFFFF, 0xCAFFFFFF, 0x40000000, 0x40400000, 0xC0400000, 0x42F60000,
	// The VFPU's ranges: sin and cos near pi / 2, the clamps at -1 and 1 and just inside them.
	0x3FC90FDB, 0x3F7FFFFF, 0xBF7FFFFF, 0x3F800001, 0xBF800001,
};

u32 GprValue(Rng &r) {
	return r.Chance(60) ? kEdge[r.Below(sizeof(kEdge) / 4)] : r.Next();
}
// Mostly floats at the edges; some integers (FCvtSW and the like read FPRs as ints), some anything.
u32 FprValue(Rng &r) {
	if (r.Chance(60))
		return kFloatEdge[r.Below(sizeof(kFloatEdge) / 4)];
	if (r.Chance(30))
		return kEdge[r.Below(sizeof(kEdge) / 4)];
	return r.Next();
}

// The scratch RAM window: loads and stores land at [W, W + kWindow), masked; base registers r28, r29
// are never written, and each access's constant is chosen from their values to land there.
constexpr u32 kWindowAt = 0x00010000;
constexpr u32 kWindowSize = 0x10000;
constexpr u8 kBaseRegA = 28, kBaseRegB = 29;

// The chained fuzz's programs: block i at kProgBase + kProgStride * i, kFuzzBlockBytes of guest code each
// (what an icache op must touch to invalidate it). r1..r3 hold the targets ExitToReg and SetPC take
// (block addresses, mirrors of them, addresses with no block, invalid ones), r27 the program's base for
// the icache ops; the blocks never write them otherwise.
constexpr u32 kProgBase = 0x08800000;
constexpr u32 kProgStride = 0x100;
constexpr u32 kFuzzBlockBytes = 0x40;
constexpr int kProgMaxBlocks = 12;
constexpr u32 kShadowWindow = 0x10000;  // the block shadow the chained fuzz gives its address spaces
constexpr u8 kExitRegs = 3;
constexpr u8 kProgReg = 27;
u32 ProgPc(int i) { return kProgBase + kProgStride * (u32)i; }

struct FuzzGen {
	Rng &r;
	u32 baseA, baseB;
	bool twoNans = false;  // CFG_SELFTEST_TWO_NANS
	int twoNanOps = 0;     // adds and multiplies given NaNs on both sides
	int vtempPicks = 0;    // FPR operands picked on the VFPU temps (192-207)
	int constBases = 0;    // guest accesses based on a register the block set to a constant (folded)
	int aliasReads = 0;    // FPR sources over the IR temps or VFPU control words set as GPRs right before
	// The chained fuzz: this is block `self` of a program of `blocks`.
	bool chain = false;
	int blocks = 0, self = 0;
	int cacheOps = 0;           // Interpret ops that are icache invalidations of a block of the program
	std::vector<IRInst> pre;    // what Op() needs right before the op it returns (a constant base register)

	u8 Dest() {
		static const u8 extra[] = {192, 193, 194, 195, 196, 197, 211, IRREG_LO, IRREG_HI};
		if (r.Chance(4))
			return 0;
		if (r.Chance(25))
			return extra[r.Below(sizeof(extra))];
		u8 d = (u8)(1 + r.Below(31));
		if (d == kBaseRegA || d == kBaseRegB || (chain && (d <= kExitRegs || d == kProgReg)))
			d = 30;
		return d;
	}
	u8 Src() {
		if (r.Chance(8))
			return r.Chance(50) ? kBaseRegA : kBaseRegB;
		return Dest();
	}
	// FPRs past the VFPU's 128 lie over the IR temps and VFPU control words (GPR-class 192..223), so
	// some land on the GPR-class registers Dest and Src use: an FPR write must unload their locals.
	// 192-207 are the VFPU frontend's own temps (IRVTEMP_PFX_S/T/D, IRVTEMP_0, over MIPSState::vt): the
	// prefixes, the blend's merge, sincos, the half-float conversions and vrot work there.
	u8 Fpr() {
		static const u8 alias[] = {160, 161, 162, 163, 164, 165, 179};
		if (r.Chance(10))
			return alias[r.Below(sizeof(alias))];
		if (r.Chance(5)) {
			vtempPicks++;
			return (u8)(192 + r.Below(16));
		}
		if (r.Chance(4))
			return (u8)(160 + r.Below(32));
		return (u8)r.Below(160);
	}
	u8 Vec() {
		if (r.Chance(8))
			return r.Chance(50) ? 160 : 176;  // over temps 192..195, or VFPU control 208..211 (CC)
		if (r.Chance(5)) {
			vtempPicks++;
			return (u8)(192 + 4 * r.Below(4));  // IRVTEMP_PFX_S, _PFX_T, _PFX_D, IRVTEMP_0
		}
		if (r.Chance(3))
			return (u8)(160 + 4 * r.Below(8));
		return (u8)(4 * r.Below(40));
	}
	u32 Pc() { return chain ? TargetPc() : r.Next() & ~3u; }
	// A target in the chained fuzz: this block (a loop), another, a mirror of one (the same shadow entry,
	// another PC), an address with no block, an unaligned one, one with no memory.
	u32 TargetPc() {
		const u32 pick = r.Below(100);
		const int j = (int)r.Below((u32)blocks);
		if (pick < 25)
			return ProgPc(self);
		if (pick < 60)
			return ProgPc(j);
		if (pick < 70)
			return ProgPc(j) | 0x40000000;
		if (pick < 82)
			return ProgPc(j) + kFuzzBlockBytes;
		if (pick < 90)
			return ProgPc(j) + 2;
		return 0x10000000 | (r.Next() & 0x00FFFFFC);
	}
	// A MIPS instruction for IROp::Interpret that MIPSInterpret runs on the state it is given and that
	// writes registers native ops read after it: an R-type ALU op, a shift or a lo/hi op (Int_RType3,
	// Int_ShiftType, Int_MulDivType: they write rd, or lo and hi, and add 4 to pc). rd is never a base
	// register; 0 sometimes (nothing written).
	u32 InterpretOp() {
		static const u8 kFunct[] = {0, 2, 3, 4, 6, 7, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 28, 29,
			32, 33, 34, 35, 36, 37, 38, 39, 42, 43, 44, 45, 46, 47};
		if (chain && r.Chance(50)) {
			// cache 8 (icache invalidate), imm(r27): Int_Cache invalidates the 64 bytes at a block of the
			// program (and up to 63 more), often the block running it, while it may be compiled and linked.
			cacheOps++;
			const u32 imm = kProgStride * (r.Chance(40) ? (u32)self : r.Below((u32)blocks)) + r.Below(0x40);
			return (0x2Fu << 26) | ((u32)kProgReg << 21) | (8u << 16) | imm;
		}
		const u32 funct = kFunct[r.Below(sizeof(kFunct))];
		u32 rd = r.Chance(5) ? 0 : 1 + r.Below(31);
		if (rd == kBaseRegA || rd == kBaseRegB || (chain && ((rd >= 1 && rd <= kExitRegs) || rd == kProgReg)))
			rd = 30;
		const u32 rs = r.Below(32);
		const u32 rt = r.Below(32);
		const u32 sa = r.Below(32);
		return (rs << 21) | (rt << 16) | (rd << 11) | (sa << 6) | funct;
	}
	// A base register and a constant that land in the window (top bits are masked off, so the add may
	// carry anywhere above them).
	void Mem(IRInst &in, u32 size) {
		if (r.Chance(25)) {
			// A base register the block set to a constant right before: the compiled code folds the address.
			const u8 reg = Dest();
			u32 value = r.Next();
			const u32 target = kWindowAt + r.Below(kWindowSize - 16 - size);
			// Built as the frontend's code builds them, sometimes in two steps (a *Const op or a move on the
			// constant), which the compiler follows.
			const u32 k = r.Next();
			switch (r.Below(10)) {
			case 0: pre.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, value - k}); pre.push_back(IRInst{IROp::AddConst, {reg}, reg, 0, k}); break;
			case 1: pre.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, value - k}); pre.push_back(IRInst{IROp::OptAddConst, {reg}, reg, 0, k}); break;
			case 2: pre.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, value + k}); pre.push_back(IRInst{IROp::SubConst, {reg}, reg, 0, k}); break;
			case 3: pre.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, value & 0xFFFF0000}); pre.push_back(IRInst{IROp::OrConst, {reg}, reg, 0, value & 0xFFFF}); break;
			case 4: pre.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, value ^ k}); pre.push_back(IRInst{IROp::XorConst, {reg}, reg, 0, k}); break;
			case 5: {
				const u8 sh = (u8)r.Below(32);
				value = (value >> sh) << sh;
				pre.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, value >> sh});
				pre.push_back(IRInst{IROp::ShlImm, {reg}, reg, sh, 0});
				break;
			}
			case 6: {
				const u8 from = Dest();
				pre.push_back(IRInst{IROp::SetConst, {from}, 0, 0, value});
				pre.push_back(IRInst{IROp::Mov, {reg}, from, 0, 0});
				break;
			}
			case 7:
			case 8: {
				// A constant the register held, overwritten by a base register (known to the fuzz, not to
				// the compiler), then maybe a *Const op on it: nothing may stay known.
				const bool a = r.Chance(50);
				pre.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, r.Next()});
				pre.push_back(IRInst{IROp::Mov, {reg}, a ? kBaseRegA : kBaseRegB, 0, 0});
				value = a ? baseA : baseB;
				if (r.Below(9) == 8) {
					pre.push_back(IRInst{IROp::AddConst, {reg}, reg, 0, k});
					value += k;
				}
				break;
			}
			default: pre.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, value}); break;
			}
			in.src1 = reg;
			in.constant = target - value + (r.Below(16) << 28);
			constBases++;
			return;
		}
		const bool a = r.Chance(50);
		in.src1 = a ? kBaseRegA : kBaseRegB;
		const u32 target = kWindowAt + r.Below(kWindowSize - 16 - size);
		in.constant = target - (a ? baseA : baseB) + (r.Below(16) << 28);
	}
	IRInst Op() {
		static const IROp kNative[] = {
			IROp::SetConst, IROp::Mov, IROp::Add, IROp::Sub, IROp::Neg, IROp::Not, IROp::And, IROp::Or, IROp::Xor,
			IROp::AddConst, IROp::OptAddConst, IROp::SubConst, IROp::AndConst, IROp::OrConst, IROp::XorConst,
			IROp::OptAndConst, IROp::OptOrConst, IROp::Shl, IROp::Shr, IROp::Sar, IROp::Ror, IROp::ShlImm,
			IROp::ShrImm, IROp::SarImm, IROp::RorImm, IROp::Slt, IROp::SltConst, IROp::SltU, IROp::SltUConst,
			IROp::Clz, IROp::MovZ, IROp::MovNZ, IROp::Max, IROp::Min, IROp::BSwap16, IROp::BSwap32,
			IROp::Ext8to32, IROp::Ext16to32, IROp::MtLo, IROp::MtHi, IROp::MfLo, IROp::MfHi, IROp::Mult,
			IROp::MultU, IROp::Madd, IROp::MaddU, IROp::Msub, IROp::MsubU, IROp::Load8, IROp::Load8Ext,
			IROp::Load16, IROp::Load16Ext, IROp::Load32, IROp::Load32Linked, IROp::LoadFloat, IROp::LoadVec4,
			IROp::Store8, IROp::Store16, IROp::Store32, IROp::Store32Conditional, IROp::StoreFloat,
			IROp::StoreVec4, IROp::FMov, IROp::SetConstF, IROp::Downcount, IROp::SetPC, IROp::SetPCConst,
		};
		// M2's: the FPU, Vec4, the VFPU, Div/DivU, the rounding-mode ops.
		static const IROp kFp[] = {
			IROp::Div, IROp::DivU,
			IROp::FAdd, IROp::FSub, IROp::FMul, IROp::FDiv, IROp::FMin, IROp::FMax, IROp::FSqrt, IROp::FNeg,
			IROp::FAbs, IROp::FSign, IROp::FSat0_1, IROp::FSatMinus1_1, IROp::FCvtSW, IROp::FCvtScaledSW,
			IROp::FRound, IROp::FTrunc, IROp::FCeil, IROp::FFloor, IROp::FCvtWS, IROp::FCvtScaledWS,
			IROp::FMovFromGPR, IROp::FMovToGPR, IROp::OptFCvtSWFromGPR, IROp::OptFMovToGPRShr8,
			IROp::FpCondFromReg, IROp::FpCondToReg, IROp::FpCtrlFromReg, IROp::FpCtrlToReg,
			IROp::VfpuCtrlToReg, IROp::SetCtrlVFPU, IROp::SetCtrlVFPUReg, IROp::SetCtrlVFPUFReg,
			IROp::FCmp, IROp::FCmpVfpuBit, IROp::FCmpVfpuBit, IROp::FCmpVfpuBit, IROp::FCmpVfpuAggregate, IROp::FCmovVfpuCC,
			IROp::Vec4Init, IROp::Vec4Shuffle, IROp::Vec4Blend, IROp::Vec4Mov, IROp::Vec4Add, IROp::Vec4Sub,
			IROp::Vec4Mul, IROp::Vec4Div, IROp::Vec4Scale, IROp::Vec4Neg, IROp::Vec4Abs, IROp::Vec4Min,
			IROp::Vec4Max, IROp::Vec4Slt, IROp::Vec4Sge, IROp::Vec4CvtScaledSW, IROp::Vec4VfpuDot, IROp::Vec4VfpuMultiDot,
			IROp::FSin, IROp::FCos, IROp::FRSqrt, IROp::FRecip, IROp::FAsin, IROp::FVSqrt, IROp::FExp2, IROp::FLog2,
			IROp::FSinCos, IROp::FFloatToHalf2, IROp::FHalfToFloat,
			IROp::Vec2Unpack16To31, IROp::Vec2Unpack16To32, IROp::Vec4Unpack8To32, IROp::Vec4DuplicateUpperBitsAndShift1,
			IROp::Vec4Pack31To8, IROp::Vec4Pack32To8, IROp::Vec2Pack31To16, IROp::Vec2Pack32To16, IROp::ReverseBits,
			IROp::ApplyRoundingMode, IROp::RestoreRoundingMode, IROp::UpdateRoundingMode,
		};
		// Still interpreter islands (and Vec4Init past its table's rows).
		static const IROp kIsland[] = {
			IROp::Load32Left, IROp::Load32Right, IROp::Store32Left, IROp::Store32Right, IROp::Vec4Init,
		};
		static const IROp kCondExit[] = {
			IROp::ExitToConstIfEq, IROp::ExitToConstIfNeq, IROp::ExitToConstIfGtZ, IROp::ExitToConstIfGeZ,
			IROp::ExitToConstIfLtZ, IROp::ExitToConstIfLeZ,
		};
		IRInst in{IROp::Nop, {0}, 0, 0, 0};
		pre.clear();
		if (r.Chance(4)) {
			in.op = kCondExit[r.Below(6)];
			in.src1 = Src();
			in.src2 = Src();
			in.constant = Pc();
			return in;
		}
		// A helper call: every local must be reloaded after it (UnloadAll), and pc, lo, hi change too.
		if (r.Chance(chain ? 6 : 3)) {
			in.op = IROp::Interpret;
			in.constant = InterpretOp();
			return in;
		}
		bool island = r.Chance(8);
		if (island)
			in.op = kIsland[r.Below(sizeof(kIsland) / sizeof(kIsland[0]))];
		else if (r.Chance(50))
			in.op = kFp[r.Below(sizeof(kFp) / sizeof(kFp[0]))];
		else
			in.op = kNative[r.Below(sizeof(kNative) / sizeof(kNative[0]))];
		in.dest = Dest();
		in.src1 = Src();
		in.src2 = Src();
		in.constant = r.Chance(50) ? GprValue(r) : r.Below(64);
		switch (in.op) {
		// M2's operands.
		case IROp::FAdd: case IROp::FSub: case IROp::FMul: case IROp::FDiv: case IROp::FMin: case IROp::FMax:
		case IROp::FFloatToHalf2:
			in.dest = Fpr(); in.src1 = Fpr(); in.src2 = r.Chance(10) ? in.src1 : Fpr();
			break;
		case IROp::FSqrt: case IROp::FNeg: case IROp::FAbs: case IROp::FSign: case IROp::FSat0_1: case IROp::FSatMinus1_1:
		case IROp::FCvtSW: case IROp::FRound: case IROp::FTrunc: case IROp::FCeil: case IROp::FFloor: case IROp::FCvtWS:
		case IROp::FSin: case IROp::FCos: case IROp::FRSqrt: case IROp::FRecip: case IROp::FAsin: case IROp::FVSqrt:
		case IROp::FExp2: case IROp::FLog2:
			in.dest = Fpr(); in.src1 = Fpr();
			break;
		case IROp::FSinCos: in.dest = (u8)std::min<u32>(Fpr(), 206); in.src1 = Fpr(); break;
		case IROp::FCvtScaledSW: case IROp::FCvtScaledWS: in.dest = Fpr(); in.src1 = Fpr(); in.src2 = (u8)r.Below(256); break;
		case IROp::FHalfToFloat: in.dest = Fpr(); in.src1 = Fpr(); in.src2 = (u8)r.Below(2); break;
		case IROp::FMovFromGPR: case IROp::OptFCvtSWFromGPR: in.dest = Fpr(); break;
		case IROp::FMovToGPR: case IROp::OptFMovToGPRShr8: in.src1 = Fpr(); break;
		case IROp::VfpuCtrlToReg: in.src1 = (u8)r.Below(16); break;
		case IROp::SetCtrlVFPU: case IROp::SetCtrlVFPUReg: in.dest = (u8)r.Below(16); break;
		case IROp::SetCtrlVFPUFReg: in.dest = (u8)r.Below(16); in.src1 = Fpr(); break;
		case IROp::FCmp: in.dest = (u8)r.Below(9); in.src1 = Fpr(); in.src2 = r.Chance(10) ? in.src1 : Fpr(); break;
		case IROp::FCmpVfpuBit: in.dest = (u8)r.Below(256); in.src1 = Fpr(); in.src2 = r.Chance(10) ? in.src1 : Fpr(); break;
		case IROp::FCmpVfpuAggregate: in.dest = (u8)(r.Chance(10) ? 0 : r.Below(64)); break;
		case IROp::FCmovVfpuCC: in.dest = Fpr(); in.src1 = Fpr(); in.src2 = (u8)r.Below(256); break;
		// Rows past the table's 8 go to the interpreter (reading what follows it, the same on both sides).
		case IROp::Vec4Init: in.dest = Vec(); in.src1 = (u8)(island ? 8 + r.Below(2) : r.Below(8)); break;
		case IROp::Vec4Shuffle: in.dest = Vec(); in.src1 = Vec(); in.src2 = (u8)r.Below(256); break;
		case IROp::Vec4Blend: in.dest = Vec(); in.src1 = Vec(); in.src2 = Vec(); in.constant = r.Below(16); break;
		case IROp::Vec4Mov: case IROp::Vec4Neg: case IROp::Vec4Abs: in.dest = Vec(); in.src1 = Vec(); break;
		case IROp::Vec4Add: case IROp::Vec4Sub: case IROp::Vec4Mul: case IROp::Vec4Div: case IROp::Vec4Min:
		case IROp::Vec4Max: case IROp::Vec4Slt: case IROp::Vec4Sge:
			in.dest = Vec(); in.src1 = Vec(); in.src2 = r.Chance(10) ? in.src1 : Vec();
			break;
		case IROp::Vec4Scale: in.dest = Vec(); in.src1 = Vec(); in.src2 = Fpr(); break;
		case IROp::Vec4CvtScaledSW: in.dest = Vec(); in.src1 = Vec(); in.src2 = (u8)r.Below(256); break;
		case IROp::Vec4VfpuDot: in.dest = Fpr(); in.src1 = Vec(); in.src2 = Vec(); break;
		case IROp::Vec4VfpuMultiDot:
			// A 4x4 block of VFPU registers (or of the FPRs over the IR temps and VFPU control words), a vector,
			// 1-4 terms, maybe transposed.
			in.dest = Vec(); in.src1 = (u8)(r.Chance(10) ? 160 + 4 * r.Below(5) : 32 + 4 * r.Below(29)); in.src2 = Vec();
			in.constant = (1 + r.Below(4)) | (r.Chance(50) ? IRMULTIDOT_TRANSPOSED : 0);
			break;
		case IROp::Vec2Unpack16To31: case IROp::Vec2Unpack16To32: in.dest = (u8)std::min<u32>(Fpr(), 206); in.src1 = Fpr(); break;
		case IROp::Vec4Unpack8To32: in.dest = Vec(); in.src1 = Fpr(); break;
		case IROp::Vec4DuplicateUpperBitsAndShift1: in.dest = Vec(); in.src1 = Vec(); break;
		case IROp::Vec4Pack31To8: case IROp::Vec4Pack32To8: in.dest = Fpr(); in.src1 = Vec(); break;
		case IROp::Vec2Pack31To16: case IROp::Vec2Pack32To16: in.dest = Fpr(); in.src1 = (u8)std::min<u32>(Fpr(), 206); break;
		case IROp::FpCondFromReg: case IROp::FpCtrlFromReg: break;
		case IROp::OptAddConst: case IROp::OptAndConst: case IROp::OptOrConst:
			in.src1 = in.dest;
			break;
		case IROp::ShlImm: case IROp::ShrImm: case IROp::SarImm: case IROp::RorImm:
			in.src2 = (u8)r.Below(32);
			break;
		case IROp::Load8: case IROp::Load8Ext: Mem(in, 1); break;
		case IROp::Load16: case IROp::Load16Ext: Mem(in, 2); break;
		case IROp::Load32: case IROp::Load32Linked: case IROp::Load32Left: case IROp::Load32Right: Mem(in, 4); break;
		case IROp::LoadFloat: in.dest = Fpr(); Mem(in, 4); break;
		case IROp::LoadVec4: in.dest = Vec(); Mem(in, 16); break;
		case IROp::Store8: case IROp::Store16: case IROp::Store32: case IROp::Store32Conditional:
		case IROp::Store32Left: case IROp::Store32Right:
			in.src3 = r.Chance(30) ? Src() : Dest();
			if (in.op == IROp::Store32Conditional && (in.src3 == kBaseRegA || in.src3 == kBaseRegB))
				in.src3 = 30;  // it writes the register it stores
			Mem(in, 4);
			break;
		case IROp::StoreFloat: in.src3 = Fpr(); Mem(in, 4); break;
		case IROp::StoreVec4: in.dest = Vec(); Mem(in, 16); break;
		case IROp::FMov: in.dest = Fpr(); in.src1 = Fpr(); break;
		case IROp::SetConstF: in.dest = Fpr(); in.constant = FprValue(r); break;
		case IROp::Downcount: in.constant = r.Below(40); break;
		case IROp::SetPCConst: in.constant = Pc(); break;
		case IROp::SetPC:
			if (chain)
				in.src1 = (u8)(1 + r.Below(kExitRegs));
			break;
		default: break;
		}
		return in;
	}
	// The values the FPU's and the VFPU's rules turn on (inf * 0 in FMul, -0 in FSqrt and the clamps, NaNs
	// in the compares...), planted in an op's sources by a SetConstF right before it, far more often than
	// the registers' random values hold them in the right places.
	void Plant(std::vector<IRInst> &ir, u32 f, int lanes) {
		static const u32 kSpecial[] = {0, 0x80000000, 0x7F800000, 0xFF800000, 0x7FC00000, 0x7F800001, 0xFFC00001,
			0x3F800000, 0xBF800000, 0x00000001, 0x80400000, 0x7F7FFFFF, 0x3F000000, 0xBFC00000};
		for (int k = 0; k < lanes; ++k) {
			if (f + k < 256 && r.Chance(lanes == 1 ? 45 : 30)) {
				const u32 v = r.Chance(70) ? kSpecial[r.Below(sizeof(kSpecial) / 4)] : FprValue(r);
				ir.push_back(IRInst{IROp::SetConstF, {(u8)(f + k)}, 0, 0, v});
			}
		}
	}
	// Which NaN an add or a multiply of two NaNs gives is the engine's choice in wasm (any "arithmetic NaN"),
	// and the hardware's rule looks at operand order: x86 returns the first operand's NaN; ARM64 the first
	// signalling one (quieted), else the first quiet one. So the result can rest on the order an engine puts
	// a commutative op's operands in, and engines reorder them: JSC's optimising tier sorts them by when
	// their values were made (in WebKit the tiered-up interpreter's FMul returned b's NaN where a block
	// fresh in JSC's first tier returned a's); V8's x64 optimising tier swaps them when the right operand
	// has no later use and is not a load it folds into the instruction (the handler's FMul, whose tests come
	// before its multiply: in Node 24 on x86-64 it returned b's NaN once optimised). FMul picks its NaN from
	// the operands' bits on both sides (the handler in ppsspp-3039c4e-wasmjit.patch, the lowering above), so
	// two NaNs there give one answer everywhere. FAdd, Vec4Add, Vec4Mul and Vec4Scale keep the hardware's
	// NaN: both sides compute them straight from single-use loads or on v128, which V8 13.6 was not seen to
	// reorder in any tier, on x86-64 or ARM64. By default the fuzz never asks: one source of each add and
	// multiply holds no NaN (a NaN and a number give that NaN, quieted, whatever the order). Subtract and
	// divide keep their order. In V8 CFG_SELFTEST_TWO_NANS asks (PlantNans): a lowering whose operand order
	// differs from its handler's then returns the other NaN wherever V8 keeps wasm's order (ARM64 in every
	// tier, x86-64's baseline tier), and FMul's pick is checked everywhere.
	void PlantNoNan(std::vector<IRInst> &ir, u32 f, int lanes) {
		// Zeros and infinities often, for FMul's inf * 0.
		static const u32 kNumber[] = {0, 0x80000000, 0x7F800000, 0xFF800000, 0x3F800000, 0xBF800000, 0x00000001, 0x7F7FFFFF};
		for (int k = 0; k < lanes; ++k) {
			if (f + k >= 256)
				continue;
			u32 v;
			if (r.Chance(50)) {
				v = kNumber[r.Below(sizeof(kNumber) / 4)];
			} else {
				do {
					v = FprValue(r);
				} while ((v & 0x7FFFFFFF) > 0x7F800000);
			}
			ir.push_back(IRInst{IROp::SetConstF, {(u8)(f + k)}, 0, 0, v});
		}
	}
	// A GPR source set to one of pool's values (never a base register: the accesses would leave the window).
	void PlantGpr(std::vector<IRInst> &ir, u8 reg, std::initializer_list<u32> pool) {
		if (reg == kBaseRegA || reg == kBaseRegB)
			return;
		ir.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, pool.begin()[r.Below((u32)pool.size())]});
	}
	// The FPRs an op writes: first and how many (0 for none).
	static std::pair<u32, int> FprDest(const IRInst &in) {
		switch (in.op) {
		case IROp::FAdd: case IROp::FSub: case IROp::FMul: case IROp::FDiv: case IROp::FMin: case IROp::FMax:
		case IROp::FSqrt: case IROp::FNeg: case IROp::FAbs: case IROp::FSign: case IROp::FSat0_1: case IROp::FSatMinus1_1:
		case IROp::FCvtSW: case IROp::FCvtScaledSW: case IROp::FRound: case IROp::FTrunc: case IROp::FCeil: case IROp::FFloor:
		case IROp::FCvtWS: case IROp::FCvtScaledWS: case IROp::FMovFromGPR: case IROp::OptFCvtSWFromGPR:
		case IROp::FSin: case IROp::FCos: case IROp::FRSqrt: case IROp::FRecip: case IROp::FAsin: case IROp::FVSqrt:
		case IROp::FExp2: case IROp::FLog2: case IROp::FFloatToHalf2: case IROp::FHalfToFloat: case IROp::FCmovVfpuCC:
		case IROp::Vec4VfpuDot: case IROp::FMov: case IROp::SetConstF: case IROp::LoadFloat:
		case IROp::Vec4Pack31To8: case IROp::Vec4Pack32To8: case IROp::Vec2Pack31To16: case IROp::Vec2Pack32To16:
			return {in.dest, 1};
		case IROp::FSinCos: case IROp::Vec2Unpack16To31: case IROp::Vec2Unpack16To32:
			return {in.dest, 2};
		case IROp::Vec4Init: case IROp::Vec4Shuffle: case IROp::Vec4Blend: case IROp::Vec4Mov: case IROp::Vec4Add:
		case IROp::Vec4Sub: case IROp::Vec4Mul: case IROp::Vec4Div: case IROp::Vec4Scale: case IROp::Vec4Neg:
		case IROp::Vec4Abs: case IROp::Vec4Min: case IROp::Vec4Max: case IROp::Vec4Slt: case IROp::Vec4Sge:
		case IROp::Vec4CvtScaledSW: case IROp::Vec4VfpuMultiDot: case IROp::LoadVec4: case IROp::Vec4Unpack8To32:
		case IROp::Vec4DuplicateUpperBitsAndShift1:
			return {in.dest, 4};
		default:
			return {0, 0};
		}
	}
	// The FPRs an op reads, as spans {first, count}: the planted ops' sources and the C helpers' (CallFpr).
	static int FprReads(const IRInst &in, std::pair<u32, u32> out[2]) {
		switch (in.op) {
		case IROp::FAdd: case IROp::FSub: case IROp::FMul: case IROp::FDiv: case IROp::FMin: case IROp::FMax:
		case IROp::FCmp: case IROp::FCmpVfpuBit: case IROp::FFloatToHalf2:
			out[0] = {in.src1, 1}; out[1] = {in.src2, 1};
			return 2;
		case IROp::FSqrt: case IROp::FNeg: case IROp::FAbs: case IROp::FSign: case IROp::FSat0_1: case IROp::FSatMinus1_1:
		case IROp::FCvtSW: case IROp::FCvtScaledSW: case IROp::FCvtScaledWS: case IROp::FRound: case IROp::FTrunc:
		case IROp::FCeil: case IROp::FFloor: case IROp::FCvtWS: case IROp::FSin: case IROp::FCos: case IROp::FRSqrt:
		case IROp::FRecip: case IROp::FAsin: case IROp::FVSqrt: case IROp::FExp2: case IROp::FLog2: case IROp::FSinCos:
		case IROp::FHalfToFloat: case IROp::FMovToGPR: case IROp::SetCtrlVFPUFReg: case IROp::FMov:
			out[0] = {in.src1, 1};
			return 1;
		case IROp::Vec4Add: case IROp::Vec4Sub: case IROp::Vec4Mul: case IROp::Vec4Div: case IROp::Vec4Min:
		case IROp::Vec4Max: case IROp::Vec4Slt: case IROp::Vec4Sge: case IROp::Vec4Blend: case IROp::Vec4VfpuDot:
			out[0] = {in.src1, 4}; out[1] = {in.src2, 4};
			return 2;
		case IROp::Vec4Scale:
			out[0] = {in.src1, 4}; out[1] = {in.src2, 1};
			return 2;
		case IROp::Vec4Mov: case IROp::Vec4Neg: case IROp::Vec4Abs: case IROp::Vec4Shuffle: case IROp::Vec4CvtScaledSW:
			out[0] = {in.src1, 4};
			return 1;
		case IROp::Vec4VfpuMultiDot:
			out[0] = {in.src1, 16}; out[1] = {in.src2, 4};
			return 2;
		default:
			return 0;
		}
	}
	// Half the time an op reads an FPR over the IR temps or VFPU control words (160-191, GPR-class 192-223),
	// that register is set as a GPR right before it, so its value is only in a local, dirty: the op (a
	// lowering, or a C helper reading MIPSState) must see it. Never a NaN (in JSC two NaNs in an add or a
	// multiply give an engine-chosen one; PlantNoNan).
	void AliasSource(std::vector<IRInst> &ir, const IRInst &in) {
		std::pair<u32, u32> spans[2];
		const int n = FprReads(in, spans);
		u32 picks[24];
		int count = 0;
		for (int k = 0; k < n; ++k) {
			for (u32 f = spans[k].first; f < spans[k].first + spans[k].second && count < 24; ++f) {
				if (f >= 160 && f < 192)
					picks[count++] = f;
			}
		}
		if (!count || !r.Chance(50))
			return;
		u32 v = r.Next();
		if ((v & 0x7FFFFFFF) > 0x7F800000)
			v &= 0xFF7FFFFF;
		ir.push_back(IRInst{IROp::SetConst, {(u8)(32 + picks[r.Below((u32)count)])}, 0, 0, v});
		aliasReads++;
	}
	// A NaN in every lane: quiet and signalling, both signs, payloads high and low, so two of them mostly
	// differ and the lowering's operand order decides which comes out (ARM64: unless one signals and the
	// other does not; a vector or a few ops of the kind still show it).
	void PlantNans(std::vector<IRInst> &ir, u32 f, int lanes) {
		static const u32 kNan[] = {0x7FC00000, 0xFFC00000, 0x7FC00001, 0x7FD00F00, 0x7FFFFFFF, 0xFFFFFFFF, 0xFFE12345,
			0x7F800001, 0xFF800001, 0x7FBFFFFF, 0xFFA5A5A5, 0x7F800ABC};
		for (int k = 0; k < lanes; ++k) {
			if (f + k < 256)
				ir.push_back(IRInst{IROp::SetConstF, {(u8)(f + k)}, 0, 0, kNan[r.Below(sizeof(kNan) / 4)]});
		}
	}
	void PlantCommutative(std::vector<IRInst> &ir, u32 src1, int lanes1, u32 src2, int lanes2) {
		if (twoNans && r.Chance(50)) {
			PlantNans(ir, src1, lanes1);
			PlantNans(ir, src2, lanes2);
			twoNanOps++;
			return;
		}
		if (r.Chance(50)) {
			Plant(ir, src1, lanes1);
			PlantNoNan(ir, src2, lanes2);
		} else {
			Plant(ir, src2, lanes2);
			PlantNoNan(ir, src1, lanes1);
		}
	}
	void PlantSources(std::vector<IRInst> &ir, const IRInst &in) {
		switch (in.op) {
		case IROp::Div: case IROp::DivU:
			// Zero divisors and Div's one overflow, INT_MIN / -1.
			if (r.Chance(40))
				PlantGpr(ir, in.src1, {0x80000000, 0x80000000, 0x80000000, 0, 0xFFFFFFFF, 0xFFFF, 0x10000, 7});
			if (r.Chance(40))
				PlantGpr(ir, in.src2, {0xFFFFFFFF, 0xFFFFFFFF, 0, 0, 1, 0x80000000});
			break;
		case IROp::FAdd: case IROp::FMul:
			PlantCommutative(ir, in.src1, 1, in.src2, 1);
			break;
		case IROp::Vec4Add: case IROp::Vec4Mul:
			PlantCommutative(ir, in.src1, 4, in.src2, 4);
			break;
		case IROp::Vec4Scale:
			PlantCommutative(ir, in.src1, 4, in.src2, 1);
			break;
		case IROp::FSub: case IROp::FDiv: case IROp::FMin: case IROp::FMax:
		case IROp::FCmp: case IROp::FCmpVfpuBit: case IROp::FFloatToHalf2:
			Plant(ir, in.src1, 1);
			Plant(ir, in.src2, 1);
			break;
		case IROp::FSqrt: case IROp::FNeg: case IROp::FAbs: case IROp::FSign: case IROp::FSat0_1: case IROp::FSatMinus1_1:
		case IROp::FCvtSW: case IROp::FCvtScaledSW: case IROp::FCvtScaledWS: case IROp::FRound: case IROp::FTrunc:
		case IROp::FCeil: case IROp::FFloor: case IROp::FCvtWS: case IROp::FSin: case IROp::FCos: case IROp::FRSqrt:
		case IROp::FRecip: case IROp::FAsin: case IROp::FVSqrt: case IROp::FExp2: case IROp::FLog2: case IROp::FSinCos:
		case IROp::FHalfToFloat: case IROp::FCmovVfpuCC: case IROp::FMovToGPR: case IROp::SetCtrlVFPUFReg:
			Plant(ir, in.src1, 1);
			break;
		case IROp::Vec4Sub: case IROp::Vec4Div: case IROp::Vec4Min:
		case IROp::Vec4Max: case IROp::Vec4Slt: case IROp::Vec4Sge: case IROp::Vec4Blend: case IROp::Vec4VfpuDot:
			Plant(ir, in.src1, 4);
			Plant(ir, in.src2, 4);
			break;
		case IROp::Vec4Mov: case IROp::Vec4Neg: case IROp::Vec4Abs: case IROp::Vec4Shuffle: case IROp::Vec4CvtScaledSW:
			Plant(ir, in.src1, 4);
			break;
		default:
			break;
		}
	}
	void Block(std::vector<IRInst> &ir) {
		ir.clear();
		// Chained, every block charges something: its loops end.
		ir.push_back(IRInst{IROp::Downcount, {0}, 0, 0, chain ? 1 + r.Below(40) : r.Below(60)});
		const int n = 1 + (int)r.Below(r.Chance(20) ? 90 : 24);
		// Some blocks hit a Break, written as the frontend writes one (SetPCConst, Break): its island
		// returns pc + 4 instead of 1, so the compiled block must return there, with the Downcounts
		// pending before it charged. What follows is dead on both sides.
		const int breakAt = r.Chance(8) ? (int)r.Below((u32)n) : -1;
		for (int i = 0; i < n; ++i) {
			if (i == breakAt) {
				if (r.Chance(50))
					ir.push_back(IRInst{IROp::Downcount, {0}, 0, 0, r.Below(40)});
				ir.push_back(IRInst{IROp::SetPCConst, {0}, 0, 0, Pc()});
				if (r.Chance(50))
					ir.push_back(IRInst{IROp::Downcount, {0}, 0, 0, r.Below(40)});
				ir.push_back(IRInst{IROp::Break, {0}, 0, 0, 0});
			}
			const IRInst op = Op();
			// Sometimes a Downcount right before the helper, which the compiled code charges before the call.
			if (op.op == IROp::Interpret && r.Chance(50))
				ir.push_back(IRInst{IROp::Downcount, {0}, 0, 0, r.Below(40)});
			PlantSources(ir, op);
			AliasSource(ir, op);
			// The IR temps and VFPU control words (GPR-class 192-223) lie over FPRs 160-191: half the time an
			// op writes one of those, that register is read as a GPR right before and right after, so a
			// local the FPR write should have dropped would show its stale value. The VFPU temps (FPRs
			// 192-207, GPR-class 224-239) are read too, though no local holds them today.
			const std::pair<u32, int> written = FprDest(op);
			u32 alias = 0;
			if (written.second && r.Chance(50)) {
				const u32 f = written.first + r.Below((u32)written.second);
				if (f >= 160 && f < 208) {
					alias = 32 + f;
					ir.push_back(IRInst{IROp::Mov, {Dest()}, (u8)alias, 0, 0});
				}
			}
			ir.insert(ir.end(), pre.begin(), pre.end());
			ir.push_back(op);
			if (alias)
				ir.push_back(IRInst{r.Chance(50) ? IROp::Mov : IROp::FpCondFromReg, {Dest()}, (u8)alias, 0, 0});
		}
		switch (r.Below(4)) {
		case 0: ir.push_back(IRInst{IROp::ExitToConst, {0}, 0, 0, Pc()}); break;
		case 1: {
			// Sometimes to a register the block just set: a constant exit once compiled.
			const u8 reg = chain ? (u8)(1 + r.Below(kExitRegs)) : Src();
			if (r.Chance(20) && reg != kBaseRegA && reg != kBaseRegB)
				ir.push_back(IRInst{IROp::SetConst, {reg}, 0, 0, Pc()});
			ir.push_back(IRInst{IROp::ExitToReg, {0}, reg, 0, 0});
			break;
		}
		case 2: ir.push_back(IRInst{IROp::ExitToPC, {0}, 0, 0, 0}); break;
		default: {
			const IROp op = (IROp)((int)IROp::OptExitToConstIfEqElse + (int)r.Below(6));
			ir.push_back(IRInst{op, {0}, Src(), Src(), Pc()});
			ir.push_back(IRInst{IROp::ExitToConst, {0}, 0, 0, Pc()});
			break;
		}
		}
	}
};

std::string g_selfTestReport;

void FillState(Rng &r, MIPSState *m, u32 baseA, u32 baseB) {
	memset((void *)m, 0, sizeof(MIPSState));
	for (int i = 1; i < 32; ++i)
		m->r[i] = GprValue(r);
	m->r[kBaseRegA] = baseA;
	m->r[kBaseRegB] = baseB;
	for (int i = 0; i < 32; ++i)
		m->fi[i] = FprValue(r);
	for (int i = 0; i < 128; ++i)
		m->vi[i] = FprValue(r);
	for (int i = 0; i < 16; ++i) {
		m->t[i] = GprValue(r);
		m->vfpuCtrl[i] = GprValue(r);
		u32 v = FprValue(r);
		memcpy(&m->vt[i], &v, 4);
	}
	m->pc = r.Next() & ~3u;
	m->lo = GprValue(r);
	m->hi = GprValue(r);
	m->fcr31 = r.Next() & 0x0181FFFF;
	m->fpcond = r.Next() & 1;
	m->downcount = (int)r.Below(700) - 100;
	m->llBit = (int)(r.Next() & 1);
}

// The chained fuzz's registers on top of FillState's: the targets r1..r3 hold, the program's base, where
// it starts and how long it may run.
void FillChainState(Rng &r, MIPSState *m, int blocks, int entry) {
	const auto block = [&]() { return ProgPc((int)r.Below((u32)blocks)); };
	m->r[1] = block();
	switch (r.Below(3)) {
	case 0: m->r[2] = block(); break;
	case 1: m->r[2] = block() + kFuzzBlockBytes; break;
	default: m->r[2] = block() | 0x40000000; break;
	}
	m->r[3] = r.Chance(50) ? (0x10000000 | (r.Next() & 0x00FFFFFC)) : block() + 2;
	m->r[kProgReg] = kProgBase;
	m->pc = ProgPc(entry);
	m->downcount = (int)r.Below(400);
}

// IRJit's dispatcher loop (RunBlocksUntilDowncount) without the compiling: runs blocks from m->pc while the
// downcount lasts and the shadow has one; returns where it stopped.
u32 RunChain(MIPSState *m, const IRInst *arena) {
	const Memory::AddressSpace &space = *m->space;
	while (m->downcount >= 0) {
		if (!space.IsValid4AlignedAddress(m->pc))
			break;
		const u32 offset = *space.GetBlockShadowEntry(m->pc);
		if (offset <= Memory::BLOCK_SHADOW_HOOK_NO_JIT)
			break;
		const IRInst *inst = arena + offset;
		if (inst->op == IROp::Downcount) {
			m->downcount -= (int)inst->constant;
			inst++;
		}
		m->pc = IRInterpret(m, inst, arena);
	}
	return m->pc;
}

// The plain IR interpreter's JIT, taking the chained fuzz's blocks as IRJit::CompileBlock would add them.
class FuzzInterpJit : public MIPSComp::IRInterpreterJit {
public:
	explicit FuzzInterpJit(MIPSState *mips) : IRInterpreterJit(mips) {}
	int AddBlock(u32 addr, u32 bytes, const std::vector<IRInst> &ir) {
		const int num = blocks_.AllocateBlock((int)addr, bytes, (u32)ir.size());
		CompileTargetBlock(num, ir);
		blocks_.FinalizeBlock(num);
		FinalizeTargetBlock(num);
		return blocks_.GetBlock(num)->GetEntry();
	}
	const IRInst *Arena() const { return arena_.data(); }
};

}  // namespace

// --- the shim's side -------------------------------------------------------------------------------------

void SetWanted(bool on) {
	g_wanted = on;
	g_paCreateIRJit = &CreateJit;
}

bool Wanted() { return g_wanted; }

bool Active() { return g_jit != nullptr; }

void Frame(double lastCoreMs) {
	if (g_jit)
		g_jit->Frame(lastCoreMs);
}

void ChaosTick(MIPSState *mips) {
	if (!g_cfg.chaos || !g_jit || ++g_chaosCalls % (u32)g_cfg.chaos != 0)
		return;
	g_jit->Chaos(mips->pc);
}

int Configure(int key, int value) {
	switch (key) {
	case CFG_THRESH: g_cfg.thresh = std::max(1, std::min(value, 0xFFFF)); return 0;
	case CFG_BATCH: g_cfg.batch = std::max(1, std::min(value, 256)); return 0;
	case CFG_ISLAND_FACTOR: g_cfg.islandFactor = std::max(0, value); return 0;
	case CFG_MAX_BLOCKS: g_cfg.maxBlocks = std::max(0, value); return 0;
	case CFG_BUDGET_MAX_US: g_cfg.budgetMaxUs = std::max(100, value); return 0;
	case CFG_MAX_MODULES: g_cfg.maxModules = std::max(1, value); return 0;
	case CFG_CANARY: g_cfg.canary = value ? 1 : 0; return 0;
	case CFG_CANARY_WINDOW: g_cfg.canaryWindow = std::max(10, value); return 0;
	case CFG_TRIP_CANARY:
		if (!g_jit)
			return -1;
		Stop("canary");
		g_jit->Revert(g_canary.certified);
		return 0;
	case CFG_SELFTEST_WARM: g_cfg.selftestWarm = std::max(0, std::min(value, 1000000)); return 0;
	case CFG_SELFTEST_TWO_NANS: g_cfg.selftestTwoNans = value ? 1 : 0; return 0;
	case CFG_SELFTEST_CHAIN: g_cfg.selftestChain = value ? 1 : 0; return 0;
	case CFG_LINKS: g_cfg.links = value & (LINK_SELF | LINK_DIRECT | LINK_TABLE | LINK_REG); return 0;
	case CFG_TAIL_CALLS: g_cfg.tailCalls = value ? 1 : 0; return 0;
	case CFG_WRITE_BACK: g_cfg.writeBack = value ? 1 : 0; return 0;
	case CFG_FOLD_CONST: g_cfg.foldConst = value ? 1 : 0; return 0;
	case CFG_COUNT_LINKS: g_cfg.countLinks = value ? 1 : 0; return 0;
	case CFG_TEST_CHAOS: g_cfg.chaos = std::max(0, value); return 0;
	case CFG_TEST_CLEAR:
		if (!g_jit)
			return -1;
		g_jit->ClearCache();
		return 0;
	default: return -1;
	}
}

void ResetWindow() {
	g_stats.callWasmBase = g_paCallWasmCount;
	g_stats.blockEntryBase = g_paBlockEntryCount;
	g_stats.compileMs = 0;
	g_stats.compileMsMax = 0;
	g_stats.framesCompiling = 0;
	g_stats.frames = 0;
	for (auto &c : g_islandExec)
		c = 0;
	for (auto &c : g_linkCount)
		c = 0;
}

const char *StatsJson() {
	static std::string out;
	char buf[512];
	out.clear();
	snprintf(buf, sizeof(buf),
		"{\"wanted\":%d,\"active\":%d,\"stopped\":\"%s\",\"tailCalls\":%d,\"blocks\":%llu,\"liveBlocks\":%u,\"hotQueued\":%u,"
		"\"modules\":%llu,\"liveModules\":%u,\"bytes\":%llu,\"rejected\":%llu,\"rejectedNoExit\":%llu,\"invalidated\":%llu,"
		"\"clears\":%llu,\"flushes\":%llu,\"freedSlots\":%llu,\"rangeErrors\":%llu,\"loadErrors\":%llu,",
		g_wanted ? 1 : 0, g_jit ? 1 : 0, g_stopReason ? g_stopReason : "", g_rt.tailCalls ? 1 : 0,
		(unsigned long long)g_stats.blocks, g_jit ? (unsigned)g_jit->LiveBlocks() : 0u, g_jit ? (unsigned)g_jit->HotQueued() : 0u,
		(unsigned long long)g_stats.modules, (unsigned)g_rt.modules.size(), (unsigned long long)g_stats.bytes,
		(unsigned long long)g_stats.rejected, (unsigned long long)g_stats.rejectedNoExit, (unsigned long long)g_stats.invalidated,
		(unsigned long long)g_stats.clears, (unsigned long long)g_stats.flushes, (unsigned long long)g_stats.freedSlots,
		(unsigned long long)g_stats.rangeErrors, (unsigned long long)g_stats.loadErrors);
	out += buf;
	snprintf(buf, sizeof(buf),
		"\"frames\":%llu,\"compileMs\":%.3f,\"compileMsMaxFrame\":%.3f,\"framesCompiling\":%llu,\"callWasm\":%u,\"blockEntry\":%u,"
		"\"canaryBaseNs\":%.2f,\"canaryLastNs\":%.2f,\"canaryWindows\":%llu,\"canaryTimings\":%llu,\"canaryStrikes\":%llu,"
		"\"canaryCoarse\":%d,\"instances\":%llu,\"reverted\":%llu,\"chaos\":%llu,\"linkSlots\":%u,",
		(unsigned long long)g_stats.frames, g_stats.compileMs, g_stats.compileMsMax, (unsigned long long)g_stats.framesCompiling,
		g_paCallWasmCount - g_stats.callWasmBase, g_paBlockEntryCount - g_stats.blockEntryBase,
		g_stats.canaryBaseNs, g_stats.canaryLastNs, (unsigned long long)g_stats.canaryWindows,
		(unsigned long long)g_stats.canaryTimings, (unsigned long long)g_stats.canaryStrikes, g_canary.coarse ? 1 : 0,
		(unsigned long long)g_stats.instances, (unsigned long long)g_stats.reverted, (unsigned long long)g_stats.chaos,
		g_jit ? (unsigned)g_jit->LinkSlots() : 0u);
	out += buf;
	// The links compiled code took in this window (with pa_jit_config 17 when it was compiled; the stub's
	// always): loops, direct calls, link-slot calls and of those the ones the stub returned from,
	// ExitToReg/ExitToPC lookups that found compiled code and that did not, and entries into compiled blocks
	// (callWasm + direct + table - stub + reg; a link that ran out of depth, without tail calls, is none).
	snprintf(buf, sizeof(buf), "\"links\":{\"self\":%u,\"direct\":%u,\"table\":%u,\"stub\":%u,\"reg\":%u,\"regMiss\":%u,\"enter\":%u},",
		g_linkCount[LC_SELF], g_linkCount[LC_DIRECT], g_linkCount[LC_TABLE], g_linkCount[LC_STUB], g_linkCount[LC_REG], g_linkCount[LC_REGMISS],
		g_linkCount[LC_ENTER]);
	out += buf;
	// Islands by their first op: [runs compiled, runs executed in this window]; rejections: [blocks, entries].
	out += "\"islands\":{";
	bool first = true;
	for (int op = 0; op < 256; ++op) {
		if (!g_stats.islands[op].blocks && !g_islandExec[op])
			continue;
		snprintf(buf, sizeof(buf), "%s\"%s\":[%llu,%u]", first ? "" : ",", OpName((IROp)op), (unsigned long long)g_stats.islands[op].blocks, g_islandExec[op]);
		out += buf;
		first = false;
	}
	out += "},\"rejectedBy\":{";
	first = true;
	for (int op = 0; op < 256; ++op) {
		if (!g_stats.rejectedBy[op].blocks)
			continue;
		snprintf(buf, sizeof(buf), "%s\"%s\":[%llu,%llu]", first ? "" : ",", OpName((IROp)op), (unsigned long long)g_stats.rejectedBy[op].blocks,
			(unsigned long long)g_stats.rejectedBy[op].exec);
		out += buf;
		first = false;
	}
	out += "},\"lastError\":\"";
	for (char c : g_stats.lastError)
		out += (c == '"' || c == '\\' || (unsigned char)c < 0x20) ? ' ' : c;
	out += "\"}";
	return out.c_str();
}

// Core_BreakException obeys the user's setting: while the fuzz runs, its Break ops only log.
struct BreaksIgnored {
	const int saved = g_Config.iExceptionActionBreak;
	BreaksIgnored() { g_Config.iExceptionActionBreak = (int)ExceptionAction::Ignore; }
	~BreaksIgnored() { g_Config.iExceptionActionBreak = saved; }
};

int SelfTestChain(uint32_t seed, int programs);

int SelfTest(uint32_t seed, int n) {
	if (g_cfg.selftestChain)
		return SelfTestChain(seed, n);
	InitIR();  // the op names in the report
	BreaksIgnored breaksIgnored;
	g_selfTestReport.clear();
	if (!g_rt.Init()) {
		g_selfTestReport = "no function table";
		return -1;
	}
	Rng r{seed ? seed : 1};
	// Two copies of everything: A runs in IRInterpret, B as compiled code.
	alignas(16) static u8 stateA[sizeof(MIPSState)], stateB[sizeof(MIPSState)], start[sizeof(MIPSState)];
	MIPSState *a = reinterpret_cast<MIPSState *>(stateA);
	MIPSState *b = reinterpret_cast<MIPSState *>(stateB);
	MIPSState *s0 = reinterpret_cast<MIPSState *>(start);
	std::vector<u8> ramA(kWindowSize + 64), ramB(kWindowSize + 64), ram0(kWindowSize + 64);
	Memory::AddressSpace spaceA, spaceB;
	spaceA.base = ramA.data() - kWindowAt;
	spaceB.base = ramB.data() - kWindowAt;
	// Warm-up calls (CFG_SELFTEST_WARM) run on a third state, where nothing is compared (their islands use a
	// third RAM; the compiled code's own accesses land in B's, which is filled again after): the base
	// registers are never written, so every access stays in the window.
	alignas(16) static u8 stateW[sizeof(MIPSState)];
	MIPSState *w = reinterpret_cast<MIPSState *>(stateW);
	std::vector<u8> ramW(kWindowSize + 64);
	Memory::AddressSpace spaceW;
	spaceW.base = ramW.data() - kWindowAt;
	CompileCtx ctx;
	ctx.memBase = (u32)(uintptr_t)spaceB.base;
	ctx.plainLoad16 = true;  // the window is nowhere near VRAM's depth mirrors
	ctx.writeBack = g_cfg.writeBack != 0;
	ctx.foldConst = g_cfg.foldConst != 0;

	int failures = 0, ran = 0, compiled = 0, withBreak = 0, interprets = 0, twoNanOps = 0, vtempPicks = 0, constBases = 0, aliasReads = 0;
	bool opSeen[256] = {};
	std::vector<std::vector<IRInst>> irs;
	std::vector<Plan> plans;
	std::vector<u32> seeds;
	while (ran < n) {
		const int count = std::min(64, n - ran);
		irs.assign(count, {});
		plans.clear();
		seeds.clear();
		for (int i = 0; i < count; ++i) {
			const u32 blockSeed = r.Next() | 1;
			seeds.push_back(blockSeed);
			Rng br{blockSeed};
			const u32 baseA = br.Next(), baseB = br.Next();
			FuzzGen gen{br, baseA, baseB};
			gen.twoNans = g_cfg.selftestTwoNans != 0;
			gen.Block(irs[i]);
			twoNanOps += gen.twoNanOps;
			vtempPicks += gen.vtempPicks;
			constBases += gen.constBases;
			aliasReads += gen.aliasReads;
			for (const IRInst &in : irs[i]) {
				withBreak += in.op == IROp::Break;
				interprets += in.op == IROp::Interpret;
				opSeen[(int)in.op] = true;
			}
			Plan p;
			if (!MakePlan(irs[i].data(), (int)irs[i].size(), true, p)) {
				g_selfTestReport += "a fuzz block did not plan\n";
				return -1;
			}
			plans.push_back(std::move(p));
		}
		std::vector<int> slots;
		int moduleId = 0;
		const int loaded = CompileModule(plans, ctx, slots, moduleId, nullptr);
		if (loaded != 1) {
			g_selfTestReport += "the fuzz module did not load: " + g_stats.lastError + "\n";
			return -1;
		}
		compiled += count;
		for (int i = 0; i < count; ++i) {
			Rng br{seeds[i]};
			const u32 baseA = br.Next(), baseB = br.Next();
			Rng sr{seeds[i] ^ 0xA5A5A5A5u};
			FillState(sr, s0, baseA, baseB);
			for (auto &byte : ram0)
				byte = (u8)sr.Next();
			if (g_cfg.selftestWarm > 0) {
				memcpy(stateW, start, sizeof(MIPSState));
				memcpy(ramW.data(), ram0.data(), ram0.size());
				w->space = &spaceW;
				const BlockFn fn = (BlockFn)(uintptr_t)slots[i];
				for (int k = 0; k < g_cfg.selftestWarm; ++k)
					fn(w);
				w->space = nullptr;
			}
			memcpy(stateA, start, sizeof(MIPSState));
			memcpy(stateB, start, sizeof(MIPSState));
			memcpy(ramA.data(), ram0.data(), ram0.size());
			memcpy(ramB.data(), ram0.data(), ram0.size());
			a->space = &spaceA;
			b->space = &spaceB;
			const u32 pcA = IRInterpret(a, irs[i].data(), nullptr);
			const u32 pcB = ((BlockFn)(uintptr_t)slots[i])(b);
			a->space = nullptr;
			b->space = nullptr;
			const bool stateSame = memcmp(stateA, stateB, sizeof(MIPSState)) == 0;
			const bool ramSame = memcmp(ramA.data(), ramB.data(), ramA.size()) == 0;
			if (pcA == pcB && stateSame && ramSame)
				continue;
			failures++;
			if (failures > 4)
				continue;
			char line[256];
			snprintf(line, sizeof(line), "block %d (seed %08x): pc %08x vs %08x%s%s\n", ran + i, seeds[i], pcA, pcB,
				stateSame ? "" : ", state differs", ramSame ? "" : ", RAM differs");
			g_selfTestReport += line;
			for (size_t at = 0; !stateSame && at < sizeof(MIPSState); at += 4) {
				u32 x, y, z;
				memcpy(&x, stateA + at, 4);
				memcpy(&y, stateB + at, 4);
				memcpy(&z, start + at, 4);
				if (x != y) {
					snprintf(line, sizeof(line), "  state word %u (byte %u): interpreter %08x, wasm %08x, before %08x\n", (unsigned)(at / 4), (unsigned)at, x, y, z);
					g_selfTestReport += line;
				}
			}
			for (size_t at = 0; !ramSame && at < ramA.size(); ++at) {
				if (ramA[at] != ramB[at]) {
					snprintf(line, sizeof(line), "  RAM %08x: interpreter %02x, wasm %02x\n", (unsigned)(kWindowAt + at), ramA[at], ramB[at]);
					g_selfTestReport += line;
					break;
				}
			}
			for (const IRInst &in : irs[i]) {
				char text[256];
				DisassembleIR(text, sizeof(text), in);
				g_selfTestReport += "    ";
				g_selfTestReport += text;
				g_selfTestReport += "\n";
			}
		}
		for (int s : slots)
			g_rt.Release(s);
		ran += count;
	}
	// Which ops the blocks had: how many the JIT lowers, how many it runs as islands.
	int nativeSeen = 0, islandSeen = 0;
	for (int op = 0; op < 256; ++op) {
		if (!opSeen[op])
			continue;
		const Kind kind = Classify((IROp)op, true);
		nativeSeen += kind == Kind::Native || kind == Kind::Helper;
		islandSeen += kind == Kind::Island;
	}
	char tail[448];
	snprintf(tail, sizeof(tail), "%d blocks, %d compiled, %d differed (%d with a Break, %d Interpret ops; %d lowered and %d island ops seen; "
		"%d adds and multiplies with NaNs on both sides, %d operands on the VFPU temps, %d accesses on a constant base, "
		"%d FPR sources set as GPRs right before)\n",
		ran, compiled, failures, withBreak, interprets, nativeSeen, islandSeen, twoNanOps, vtempPicks, constBases, aliasReads);
	g_selfTestReport += tail;
	return failures;
}

// The chained fuzz (pa_jit_config 12): programs of 2..12 random blocks at fixed addresses that exit to each
// other, loop, exit through registers and the PC, to mirrors of their addresses, to addresses with no
// block and to invalid ones, and run the guest's icache op on blocks of the program while they run. One copy
// runs on the plain IR interpreter's JIT, the other on a real IRWasmJit with most blocks compiled in modules
// of random sizes (so exits are loops, direct calls, link slots and lookups), each through IRJit's
// dispatcher loop from the same state; state, RAM, the block shadows and the PC they stop at must match.
// Each program runs three times; mostly a block with other IR is then added at a mirror of one's address
// (it takes over that shadow entry) and it runs twice more; then both JITs clear their caches and take
// other code at the same addresses (a new program, or the same blocks at the same arena offsets doing
// other things: a link left over would run old code), which runs twice. Half the programs link with plain calls (no tail calls), a quarter write registers
// through. Returns how many runs differed.
int SelfTestChain(uint32_t seed, int programs) {
	InitIR();
	BreaksIgnored breaksIgnored;
	g_selfTestReport.clear();
	if (!g_rt.Init()) {
		g_selfTestReport = "no function table";
		return -1;
	}
	// The fuzz's JIT is a real one: what it changes of the session's is put back after.
	const Config savedCfg = g_cfg;
	const Stats savedStats = g_stats;
	const char *savedStop = g_stopReason;
	IRWasmJit *savedJit = g_jit;
	const u32 savedArena = g_arenaCell;
	u32 links0[LC_COUNT];
	memcpy(links0, g_linkCount, sizeof(links0));
	g_cfg.thresh = 128;  // no neighbours pulled into a batch, no block hot within a run
	g_cfg.islandFactor = 0;
	g_cfg.maxBlocks = 1 << 30;
	g_cfg.links = LINK_SELF | LINK_DIRECT | LINK_TABLE | LINK_REG;
	g_cfg.countLinks = 1;
	g_cfg.chaos = 0;
	g_stopReason = nullptr;

	Rng r{seed ? seed : 1};
	// The three states on the heap, 16-aligned: static buffers here would move the core's other globals
	// (Chrome ran the JIT-off race 3-4% slower with ~10 KB more of .bss in front of them).
	std::vector<u8> stateBytes(3 * sizeof(MIPSState) + 16);
	u8 *stateA = reinterpret_cast<u8 *>((reinterpret_cast<uintptr_t>(stateBytes.data()) + 15) & ~(uintptr_t)15);
	u8 *stateB = stateA + sizeof(MIPSState);
	u8 *start = stateB + sizeof(MIPSState);
	static_assert(sizeof(MIPSState) % 16 == 0, "the states stay 16-aligned one after another");
	MIPSState *a = reinterpret_cast<MIPSState *>(stateA);
	MIPSState *b = reinterpret_cast<MIPSState *>(stateB);
	MIPSState *s0 = reinterpret_cast<MIPSState *>(start);
	std::vector<u8> ramA(kWindowSize + 64), ramB(kWindowSize + 64), ram0(kWindowSize + 64);
	std::vector<u8> shadowA(kShadowWindow), shadowB(kShadowWindow);
	Memory::AddressSpace spaceA, spaceB;
	spaceA.base = ramA.data() - kWindowAt;
	spaceB.base = ramB.data() - kWindowAt;
	spaceA.blockShadow = shadowA.data() - kProgBase;
	spaceB.blockShadow = shadowB.data() - kProgBase;

	int failures = 0, runs = 0, compiledBlocks = 0, cacheOps = 0, noTail = 0, writeThrough = 0, blocksTotal = 0, mirrors = 0;
	const u64 invalidated0 = g_stats.invalidated;
	for (int prog = 0; prog < programs; ++prog) {
		Rng pr{r.Next() | 1};
		const int blocks = 2 + (int)pr.Below(kProgMaxBlocks - 1);
		const u32 baseA = pr.Next(), baseB = pr.Next();
		std::vector<std::vector<IRInst>> irs(blocks);
		const auto generate = [&]() {
			for (int i = 0; i < blocks; ++i) {
				FuzzGen gen{pr, baseA, baseB};
				gen.chain = true;
				gen.blocks = blocks;
				gen.self = i;
				gen.twoNans = savedCfg.selftestTwoNans != 0;
				gen.Block(irs[i]);
				cacheOps += gen.cacheOps;
			}
			blocksTotal += blocks;
		};
		generate();
		std::vector<bool> compile(blocks);
		for (int i = 0; i < blocks; ++i)
			compile[i] = pr.Chance(85);
		g_cfg.batch = 1 + (int)pr.Below((u32)blocks);
		g_cfg.tailCalls = (prog & 1) ? savedCfg.tailCalls : 0;
		g_cfg.writeBack = (prog & 3) == 3 ? 0 : savedCfg.writeBack;
		noTail += g_cfg.tailCalls == 0 || !g_rt.tailCalls;
		writeThrough += g_cfg.writeBack == 0;

		memset(stateA, 0, sizeof(MIPSState));
		memset(stateB, 0, sizeof(MIPSState));
		a->space = &spaceA;
		b->space = &spaceB;
		std::fill(shadowA.begin(), shadowA.end(), 0);
		std::fill(shadowB.begin(), shadowB.end(), 0);
		bool stop = false;
		int mirrorOf = -1;
		std::vector<IRInst> mirrorIr;
		{
			FuzzInterpJit interp(a);
			IRWasmJit jit(b);
			for (int round = 0; round < 2 && !stop; ++round) {
				if (round == 1) {
					// Other code at the same addresses: a new program, or (half the time) each block as long
					// as before, so at the same arena offset, with another downcount and other exit targets.
					// A link left from round 0 would run the old code.
					interp.ClearCache();
					jit.ClearCache();
					if (pr.Chance(50)) {
						generate();
					} else {
						for (int i = 0; i < blocks; ++i) {
							FuzzGen gen{pr, baseA, baseB};
							gen.chain = true;
							gen.blocks = blocks;
							gen.self = i;
							irs[i][0].constant = 1 + pr.Below(40);
							for (IRInst &in : irs[i]) {
								const IROp op = in.op;
								if (op == IROp::ExitToConst || op == IROp::SetPCConst || IsElseExit(op) ||
									(op >= IROp::ExitToConstIfEq && op <= IROp::ExitToConstIfLeZ))
									in.constant = gen.TargetPc();
							}
						}
					}
					for (int i = 0; i < blocks; ++i)
						compile[i] = pr.Chance(85);
				}
				std::vector<int> toCompile;
				for (int i = 0; i < blocks; ++i) {
					const int ea = interp.AddBlock(ProgPc(i), kFuzzBlockBytes, irs[i]);
					const int eb = jit.TestAddBlock(ProgPc(i), kFuzzBlockBytes, irs[i]);
					if (ea != eb) {
						g_selfTestReport += "the two arenas laid the program out differently\n";
						failures++;
						stop = true;
						break;
					}
					if (compile[i])
						toCompile.push_back(eb);
				}
				if (stop)
					break;
				compiledBlocks += jit.TestCompile(toCompile);
				for (int run = 0; run < (round == 0 ? 5 : 2); ++run) {
					if (round == 0 && run == 3) {
						// A block at a mirror of one's address, with other IR, takes over its shadow entry
						// (FinalizeBlock): what linked to the old one must reach the new one, compiled or not.
						if (!pr.Chance(75))
							break;
						mirrorOf = (int)pr.Below((u32)blocks);
						FuzzGen gen{pr, baseA, baseB};
						gen.chain = true;
						gen.blocks = blocks;
						gen.self = mirrorOf;
						gen.twoNans = savedCfg.selftestTwoNans != 0;
						gen.Block(mirrorIr);
						cacheOps += gen.cacheOps;
						const u32 at = ProgPc(mirrorOf) | 0x40000000;
						const int ea = interp.AddBlock(at, kFuzzBlockBytes, mirrorIr);
						const int eb = jit.TestAddBlock(at, kFuzzBlockBytes, mirrorIr);
						if (ea != eb) {
							g_selfTestReport += "the two arenas laid the mirror block out differently\n";
							failures++;
							stop = true;
							break;
						}
						if (pr.Chance(30))
							compiledBlocks += jit.TestCompile({eb});
						mirrors++;
					}
					const int entry = (int)pr.Below((u32)blocks);
					Rng sr{pr.Next() | 1};
					FillState(sr, s0, baseA, baseB);
					FillChainState(sr, s0, blocks, entry);
					for (auto &byte : ram0)
						byte = (u8)sr.Next();
					memcpy(stateA, start, sizeof(MIPSState));
					memcpy(stateB, start, sizeof(MIPSState));
					memcpy(ramA.data(), ram0.data(), ram0.size());
					memcpy(ramB.data(), ram0.data(), ram0.size());
					a->space = &spaceA;
					b->space = &spaceB;
					a->jit = &interp;
					b->jit = &jit;
					g_depthCell = 0;
					const u32 pcA = RunChain(a, interp.Arena());
					u32 lc0[LC_COUNT];
					memcpy(lc0, g_linkCount, sizeof(lc0));
					const u32 callWasm0 = g_paCallWasmCount;
					const u32 pcB = RunChain(b, jit.TestArena());
					const u32 depthAfter = g_depthCell;
					// The counters count what ran: every entry into a compiled block came from IRInterpret's
					// CallWasm or from a link taken (a link that ran out of depth is no link).
					const auto lc = [&](int k) { return g_linkCount[k] - lc0[k]; };
					const bool countsAddUp =
						lc(LC_ENTER) == (g_paCallWasmCount - callWasm0) + lc(LC_DIRECT) + lc(LC_TABLE) - lc(LC_STUB) + lc(LC_REG);
					a->jit = nullptr;
					b->jit = nullptr;
					a->space = nullptr;
					b->space = nullptr;
					const bool stateSame = memcmp(stateA, stateB, sizeof(MIPSState)) == 0;
					const bool ramSame = memcmp(ramA.data(), ramB.data(), ramA.size()) == 0;
					const bool shadowSame = memcmp(shadowA.data(), shadowB.data(), shadowA.size()) == 0;
					a->space = &spaceA;
					b->space = &spaceB;
					runs++;
					if (pcA == pcB && stateSame && ramSame && shadowSame && depthAfter == 0 && countsAddUp)
						continue;
					failures++;
					if (failures > 3)
						continue;
					char line[320];
					snprintf(line, sizeof(line), "program %d round %d run %d (tail calls %d, write-back %d, batch %d): pc %08x vs %08x%s%s%s%s%s\n",
						prog, round, run, g_cfg.tailCalls && g_rt.tailCalls, g_cfg.writeBack, g_cfg.batch, pcA, pcB,
						stateSame ? "" : ", state differs", ramSame ? "" : ", RAM differs", shadowSame ? "" : ", block shadow differs",
						depthAfter ? ", link depth left over" : "", countsAddUp ? "" : ", the link counters do not add up to the entries");
					g_selfTestReport += line;
					// MIPSState::space, compared null on both sides, points at each side's own address space
					// again: not a difference.
					const size_t spaceAt = offsetof(MIPSState, space);
					for (size_t at = 0; !stateSame && at < sizeof(MIPSState); at += 4) {
						if (at >= spaceAt && at < spaceAt + sizeof(a->space))
							continue;
						u32 x, y, z;
						memcpy(&x, stateA + at, 4);
						memcpy(&y, stateB + at, 4);
						memcpy(&z, start + at, 4);
						if (x != y) {
							snprintf(line, sizeof(line), "  state word %u (byte %u): interpreter %08x, wasm %08x, before %08x\n", (unsigned)(at / 4), (unsigned)at, x, y, z);
							g_selfTestReport += line;
						}
					}
					for (size_t at = 0; !ramSame && at < ramA.size(); ++at) {
						if (ramA[at] != ramB[at]) {
							snprintf(line, sizeof(line), "  RAM %08x: interpreter %02x, wasm %02x\n", (unsigned)(kWindowAt + at), ramA[at], ramB[at]);
							g_selfTestReport += line;
							break;
						}
					}
					snprintf(line, sizeof(line), "  started at block %d\n", entry);
					g_selfTestReport += line;
					for (int i = 0; i <= blocks; ++i) {
						if (i == blocks && (mirrorOf < 0 || round > 0 || run < 3))
							break;
						const std::vector<IRInst> &blockIr = i < blocks ? irs[i] : mirrorIr;
						if (i < blocks)
							snprintf(line, sizeof(line), "  block %d at %08x%s:\n", i, ProgPc(i), compile[i] ? ", compiled" : "");
						else
							snprintf(line, sizeof(line), "  then a block at %08x (a mirror of block %d's address):\n", ProgPc(mirrorOf) | 0x40000000, mirrorOf);
						g_selfTestReport += line;
						for (const IRInst &in : blockIr) {
							char text[256];
							DisassembleIR(text, sizeof(text), in);
							g_selfTestReport += "    ";
							g_selfTestReport += text;
							g_selfTestReport += "\n";
						}
					}
				}
			}
		}  // ~IRWasmJit frees its slots
		a->space = nullptr;
		b->space = nullptr;
	}
	u32 links[LC_COUNT];
	for (int k = 0; k < LC_COUNT; ++k)
		links[k] = g_linkCount[k] - links0[k];
	const u64 invalidated = g_stats.invalidated - invalidated0;
	char tail[512];
	snprintf(tail, sizeof(tail),
		"%d programs, %d runs, %d differed (%d blocks, %d compiled; links: %u self, %u direct, %u table, %u stub, %u reg, %u reg misses; "
		"%d icache ops, %llu compiled blocks invalidated, %d blocks added at a mirror; %d programs without tail calls, %d writing through)\n",
		programs, runs, failures, blocksTotal, compiledBlocks, links[LC_SELF], links[LC_DIRECT], links[LC_TABLE], links[LC_STUB], links[LC_REG],
		links[LC_REGMISS], cacheOps, (unsigned long long)invalidated, mirrors, noTail, writeThrough);
	g_selfTestReport += tail;

	g_cfg = savedCfg;
	g_stats = savedStats;
	g_stopReason = savedStop;
	g_jit = savedJit;
	// The fuzz's JIT left the hook null; a JIT of the session's gets it back. Only ever stores of &OnHot or
	// null, as everywhere else: then -flto calls the hook directly in IRInterpret's BlockEntry and inlines
	// it, as in M2. (Restoring a saved value, or a select of the two, changed IRInterpret's code, and the
	// JIT-off interpreter ran the race 2-3% slower in both engines.)
	if (savedJit)
		g_paWasmJitHot = &IRWasmJit::OnHot;
	g_arenaCell = savedArena;
	memcpy(g_linkCount, links0, sizeof(links0));
	return failures;
}

const char *SelfTestReport() { return g_selfTestReport.c_str(); }

}  // namespace PaWasmJit
