diff --git a/src/dynarmic/CMakeLists.txt b/src/dynarmic/CMakeLists.txt index b234cd2..9ffbb3c 100644 --- a/src/dynarmic/CMakeLists.txt +++ b/src/dynarmic/CMakeLists.txt @@ -340,6 +340,22 @@ if ("x86_64" IN_LIST ARCHITECTURE) endif() endif() +if ("wasm" IN_LIST ARCHITECTURE) + if ("A32" IN_LIST DYNARMIC_FRONTENDS) + target_architecture_specific_sources(dynarmic "wasm" + backend/wasm/a32_interface.cpp + backend/wasm/a32_jitstate.cpp + backend/wasm/a32_jitstate.h + backend/wasm/emit_a32.cpp + backend/wasm/emit_a32.h + backend/wasm/exclusive_monitor.cpp + backend/wasm/fp_soft.cpp + backend/wasm/fp_soft.h + backend/wasm/wasm_builder.h + ) + endif() +endif() + if ("arm64" IN_LIST ARCHITECTURE) target_link_libraries(dynarmic PRIVATE merry::oaknut) @@ -485,6 +501,8 @@ elseif (APPLE) backend/exception_handler_macos_mig.c ) endif() +elseif ("wasm" IN_LIST ARCHITECTURE) + # no hardware fault handling / fastmem on WebAssembly elseif (UNIX) if (CMAKE_SYSTEM_NAME STREQUAL "Linux") target_link_libraries(dynarmic PRIVATE rt) diff --git a/src/dynarmic/backend/wasm/a32_interface.cpp b/src/dynarmic/backend/wasm/a32_interface.cpp new file mode 100644 index 0000000..12ed228 --- /dev/null +++ b/src/dynarmic/backend/wasm/a32_interface.cpp @@ -0,0 +1,488 @@ +/* This file is part of the panda3ds-wasm project. + * SPDX-License-Identifier: GPL-3.0-or-later + * + * A32 Jit for WebAssembly: guest basic blocks are translated by Dynarmic's front end to IR, the IR is emitted as a small + * WASM module (emit_a32.cpp), and the module is instantiated through the JavaScript WebAssembly API so that the host + * engine (e.g. WebKit inside WKWebView) JIT-compiles it. Everything here runs inside an Emscripten module. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "dynarmic/backend/wasm/a32_jitstate.h" +#include "dynarmic/backend/wasm/emit_a32.h" +#include "dynarmic/backend/wasm/fp_soft.h" +#include "dynarmic/frontend/A32/a32_location_descriptor.h" +#include "dynarmic/frontend/A32/translate/a32_translate.h" +#include "dynarmic/interface/A32/a32.h" +#include "dynarmic/interface/A32/config.h" +#include "dynarmic/interface/exclusive_monitor.h" +#include "dynarmic/ir/basic_block.h" +#include "dynarmic/ir/opt/passes.h" + +// ---------- JavaScript side ---------- +EM_JS_DEPS(dynarmic_wasm_deps, "$UTF8ToString,$getWasmTableEntry"); + +EM_JS(void, jit_js_register_helper, (const char* name, int fn), { + if (!Module.jit) Module.jit = { impls: [], helpers: {} }; + Module.jit.helpers[UTF8ToString(name)] = getWasmTableEntry(fn); +}); + +EM_JS(int, jit_js_new_impl, (), { + if (!Module.jit) Module.jit = { impls: [], helpers: {} }; + Module.jit.impls.push({ table: new WebAssembly.Table({ element: 'anyfunc', initial: 256 }) }); + return Module.jit.impls.length - 1; +}); + +// Compiles a block module and installs it in the Impl's function table at `slot`. +// Every block module has the same header/type/import sections (see Emitter::Assemble), so the function entries of many blocks can be moved into ONE module: +// in WebKit each live WebAssembly.Instance costs time on every frame (59 Hz with 10 k instances, 43 Hz with 40 k, 25 Hz with 70 k), the number of functions does not. +// With batch > 0 every block first gets its own instance (it has to run now); after `batch` blocks their entries are packed into one module and the table +// slots are pointed at its exports, which leaves the single-block instances to the garbage collector. +EM_JS(int, jit_js_compile, (int id, int slot, const uint8_t* p, int n, int prefix, int off, int len, int batch), { + try { + const impl = Module.jit.impls[id]; + const tA = performance.now(); + const mod = new WebAssembly.Module(HEAPU8.slice(p, p + n)); + const tB = performance.now(); + let imports = impl.imports; + if (!prefix || !imports) { // prefix 0 = legacy layout: every module imports only what it uses, so the import object is built per module + imports = { e: { m: wasmMemory, t: impl.table, h: wasmTable } }; + const fimps = WebAssembly.Module.imports(mod).filter((x) => x.kind === 'function'); + for (const imp of fimps) { + const f = Module.jit.helpers[imp.name]; + if (!f) throw new Error('missing helper ' + imp.name); + imports.e[imp.name] = f; + } + if (prefix) { impl.imports = imports; impl.nHelpers = fimps.length; } + } + const inst = new WebAssembly.Instance(mod, imports); + const tC = performance.now(); + const T = Module.jit.t || (Module.jit.t = [0, 0, 0, 0]); T[1] += tB - tA; T[2] += tC - tB; // [0] translate+optimize+emit (C++), [1] WebAssembly.Module, [2] WebAssembly.Instance + table + while (slot >= impl.table.length) impl.table.grow(impl.table.length); + impl.table.set(slot, inst.exports.b); + if (!impl.fnCache) impl.fnCache = []; + impl.fnCache[slot] = inst.exports.b; + impl.count = (impl.count || 0) + 1; impl.bytes = (impl.bytes || 0) + n; + if (batch > 0) { + if (!impl.prefix) impl.prefix = HEAPU8.slice(p, p + prefix); + (impl.pend || (impl.pend = [])).push({ slot, ent: HEAPU8.slice(p + off, p + off + len) }); + if (impl.pend.length >= batch) { + const tD = performance.now(), pend = impl.pend; impl.pend = []; + const leb = (v) => { const o = []; do { let b = v & 0x7f; v >>>= 7; if (v) b |= 0x80; o.push(b); } while (v); return o; }; + const sec = (idn, body) => [idn, ...leb(body.length), ...body]; + const K = pend.length, nH = impl.nHelpers, enc = new TextEncoder(); + const exps = [...leb(K)]; for (let j = 0; j < K; j++) { const nm = enc.encode('b' + j); exps.push(nm.length, ...nm, 0x00, ...leb(nH + j)); } + const parts = [impl.prefix, new Uint8Array(sec(3, [...leb(K), ...new Array(K).fill(0)])), new Uint8Array(sec(7, exps))]; + let esz = 0; for (const x of pend) esz += x.ent.length; + const cbody = [...leb(K)], chead = [10, ...leb(cbody.length + esz)]; // section id, size, count, then the entries + const total = parts.reduce((a, x) => a + x.length, 0) + chead.length + cbody.length + esz; + const out = new Uint8Array(total); let o = 0; + for (const x of parts) { out.set(x, o); o += x.length; } + out.set(chead, o); o += chead.length; out.set(cbody, o); o += cbody.length; + for (const x of pend) { out.set(x.ent, o); o += x.ent.length; } + const binst = new WebAssembly.Instance(new WebAssembly.Module(out), impl.imports); + for (let j = 0; j < K; j++) { const f = binst.exports['b' + j]; impl.table.set(pend[j].slot, f); impl.fnCache[pend[j].slot] = f; } + impl.batches = (impl.batches || 0) + 1; T[3] += performance.now() - tD; // [3] packing batches into one module + } + } + return slot; + } catch (e) { + console.error('[jit] module failed:', e && e.message ? e.message : e); + return -1; + } +}); + +EM_JS(int, jit_js_run, (int id, int slot, int state), { + const impl = Module.jit.impls[id]; + const fn = (impl.fnCache && impl.fnCache[slot]) || impl.table.get(slot); + return fn(state); +}); + +// Drops all compiled blocks: a fresh table (old blocks keep a reference to the old one but are never entered again). +EM_JS(void, jit_js_clear, (int id, int dropped), { + const impl = Module.jit.impls[id]; + impl.table = new WebAssembly.Table({ element: 'anyfunc', initial: 256 }); + impl.fnCache = []; impl.imports = null; impl.pend = []; + if (!impl.flushes || impl.flushes < 3) { try { Module.printErr('[jit] cache flush #' + ((impl.flushes || 0) + 1) + ' dropping ' + dropped + ' blocks, called from: ' + (new Error().stack || '').split('\n').slice(1, 9).map((l) => { l = l.trim(); const k = l.indexOf(' ('); return k > 0 ? l.slice(3, k) : l; }).join(' < ')); } catch (e) {} } + impl.flushes = (impl.flushes || 0) + 1; impl.droppedBlocks = (impl.droppedBlocks || 0) + dropped; // diagnostics: how often the whole cache is thrown away, and how many blocks that wasted +}); + +// Time accounting for the panel: kind 0 = translate + optimize + emit (C++ side), see jit_js_compile for the other two. +EM_JS(void, jit_js_time, (int kind, double ms), { const T = Module.jit.t || (Module.jit.t = [0, 0, 0, 0]); T[kind] += ms; }); +EM_JS(double, jit_now_ms, (), { return performance.now(); }); +// Dispatch statistics for the panel/probes: S[0] = jit_js_run calls (returns to the C++ dispatcher), S[1] = guest ticks executed, S[2] = Jit::Run() calls. +EM_JS(void, jit_js_stat, (int d, double t), { const S = Module.jit.s || (Module.jit.s = [0, 0, 0, 0, 0]); S[0] += d; S[1] += t; S[2]++; }); +// S[3] = ms inside Jit::Run() (guest code + callbacks), S[4] = ms inside the SVC/exception/interpreter callbacks (Panda C++); guest code alone = S[3] - S[4]. +EM_JS(void, jit_js_stat_ms, (int i, double ms), { const S = Module.jit.s || (Module.jit.s = [0, 0, 0, 0, 0]); S[i] += ms; }); + +// Breadcrumb for hang diagnosis: the message reaches the native app even if the page then spins forever. +EM_JS(void, jit_js_crumb, (int kind, int pc, int x), { + const n = typeof window !== 'undefined' && window.webkit && window.webkit.messageHandlers && window.webkit.messageHandlers.panda; + if (n) n.postMessage(JSON.stringify({ type: 'crumb', k: kind, pc: pc >>> 0, x: x >>> 0 })); +}); + +namespace Dynarmic::A32 { + +struct Jit::Impl { + explicit Impl(UserConfig c) : conf(std::move(c)) { + static bool registered = false; + if (!registered) { + registered = true; +#define REG(n, f) do { jit_js_register_helper(n, static_cast(reinterpret_cast(&f))); for (size_t k_ = 0; k_ < std::size(Backend::Wasm::kHelpers); k_++) if (std::strcmp(Backend::Wasm::kHelpers[k_].name, n) == 0) helper_fn[k_] = static_cast(reinterpret_cast(&f)); } while (0) + REG("r8", H_r8); REG("r16", H_r16); REG("r32", H_r32); REG("r64", H_r64); + REG("w8", H_w8); REG("w16", H_w16); REG("w32", H_w32); REG("w64", H_w64); + REG("xr8", H_xr8); REG("xr16", H_xr16); REG("xr32", H_xr32); REG("xr64", H_xr64); + REG("xw8", H_xw8); REG("xw16", H_xw16); REG("xw32", H_xw32); REG("xw64", H_xw64); + REG("clrex", H_clrex); REG("svc", H_svc); REG("exc", H_exc); REG("interp", H_interp); REG("isb", H_isb); + REG("getcpsr", H_getcpsr); REG("setcpsr", H_setcpsr); REG("getfpscr", H_getfpscr); REG("setfpscr", H_setfpscr); + REG("pk", H_pk); REG("cpget1", H_cpget1); REG("cpcall", H_cpcall); + REG("fadd32", H_fadd32); REG("fsub32", H_fsub32); REG("fmul32", H_fmul32); REG("fdiv32", H_fdiv32); REG("fsqrt32", H_fsqrt32); + REG("fadd64", H_fadd64); REG("fsub64", H_fsub64); REG("fmul64", H_fmul64); REG("fdiv64", H_fdiv64); REG("fsqrt64", H_fsqrt64); + REG("fcmp32", H_fcmp32); REG("fcmp64", H_fcmp64); REG("fs2d", H_fs2d); REG("fd2s", H_fd2s); REG("ftoi", H_ftoi); REG("itof", H_itof); +#undef REG + } + st.ctx = static_cast(reinterpret_cast(this)); + js_id = jit_js_new_impl(); + } + + UserConfig conf; + inline static u32 helper_fn[std::size(Backend::Wasm::kHelpers)] = {}; // filled by REG: where each helper sits in the host's function table + Backend::Wasm::A32JitState st; + static constexpr u32 kLinkEntries = 1u << 16; // was 1 << 14: with 16k entries hot blocks evicted each other (1579 returns to the dispatcher per frame in the title scene; 553 with 64k, 320 with 256k, whose 4 MB table did not make the guest time better on the Mac) + std::unordered_map cache; // location descriptor -> slot in the function table + std::vector link = std::vector(kLinkEntries, Backend::Wasm::LinkEntry{0xFFFFFFFFu, 0, 0, 0}); + int js_id = -1; + u32 next_slot = 0; + bool invalidate_pending = false; + std::vector> slot_range; // guest [start, end) covered by the block in each table slot + std::vector> pending_ranges; // InvalidateCacheRange requests that arrived while code was running: {start, size} + std::string last_error; + + static Impl* I(u32 ctx) { return reinterpret_cast(static_cast(ctx)); } + + // ----- host functions called from generated code ----- + static u32 Crumbs() { static const u32 n = std::getenv("JIT_CRUMB") ? static_cast(std::atoi(std::getenv("JIT_CRUMB"))) : 0; return n; } + u32 crumb_n = 0; + static inline bool hc_on = false; // helper crumbs: switched on once JIT_CRUMBH blocks have been compiled (so the log is not flooded from the start) + static void HC(u32 kind, u32 va, u32 c) { if (hc_on) jit_js_crumb(10 + kind, va, I(c)->st.regs[15]); } + static bool Trace() { static const bool t = std::getenv("JIT_TRACE") != nullptr && std::getenv("JIT_TRACE")[0] == '2'; return t; } + static u32 H_r8(u32 c, u32 va) { HC(0, va, c); if (Trace()) std::fprintf(stderr, "[jit] r8 %08x pc=%08x\n", va, I(c)->st.regs[15]); return I(c)->conf.callbacks->MemoryRead8(va); } + static u32 H_r16(u32 c, u32 va) { HC(1, va, c); if (Trace()) std::fprintf(stderr, "[jit] r16 %08x\n", va); return I(c)->conf.callbacks->MemoryRead16(va); } + static u32 H_r32(u32 c, u32 va) { HC(2, va, c); if (Trace()) std::fprintf(stderr, "[jit] r32 %08x\n", va); return I(c)->conf.callbacks->MemoryRead32(va); } + static u64 H_r64(u32 c, u32 va) { HC(3, va, c); return I(c)->conf.callbacks->MemoryRead64(va); } + static void H_w8(u32 c, u32 va, u32 v) { HC(4, va, c); I(c)->conf.callbacks->MemoryWrite8(va, static_cast(v)); } + static void H_w16(u32 c, u32 va, u32 v) { HC(5, va, c); I(c)->conf.callbacks->MemoryWrite16(va, static_cast(v)); } + static void H_w32(u32 c, u32 va, u32 v) { HC(6, va, c); I(c)->conf.callbacks->MemoryWrite32(va, v); } + static void H_w64(u32 c, u32 va, u64 v) { HC(7, va, c); I(c)->conf.callbacks->MemoryWrite64(va, v); } + + template + static T ExclusiveRead(Impl* i, u32 va, RD rd) { + return i->conf.global_monitor->template ReadAndMark(i->conf.processor_id, va, rd); + } + static u32 H_xr8(u32 c, u32 va) { auto* i = I(c); return ExclusiveRead(i, va, [&] { return i->conf.callbacks->MemoryRead8(va); }); } + static u32 H_xr16(u32 c, u32 va) { auto* i = I(c); return ExclusiveRead(i, va, [&] { return i->conf.callbacks->MemoryRead16(va); }); } + static u32 H_xr32(u32 c, u32 va) { auto* i = I(c); return ExclusiveRead(i, va, [&] { return i->conf.callbacks->MemoryRead32(va); }); } + static u64 H_xr64(u32 c, u32 va) { auto* i = I(c); return ExclusiveRead(i, va, [&] { return i->conf.callbacks->MemoryRead64(va); }); } + static u32 H_xw8(u32 c, u32 va, u32 v) { auto* i = I(c); return i->conf.global_monitor->DoExclusiveOperation(i->conf.processor_id, va, [&](u8 e) { return i->conf.callbacks->MemoryWriteExclusive8(va, static_cast(v), e); }) ? 0 : 1; } + static u32 H_xw16(u32 c, u32 va, u32 v) { auto* i = I(c); return i->conf.global_monitor->DoExclusiveOperation(i->conf.processor_id, va, [&](u16 e) { return i->conf.callbacks->MemoryWriteExclusive16(va, static_cast(v), e); }) ? 0 : 1; } + static u32 H_xw32(u32 c, u32 va, u32 v) { auto* i = I(c); return i->conf.global_monitor->DoExclusiveOperation(i->conf.processor_id, va, [&](u32 e) { return i->conf.callbacks->MemoryWriteExclusive32(va, v, e); }) ? 0 : 1; } + static u32 H_xw64(u32 c, u32 va, u64 v) { auto* i = I(c); return i->conf.global_monitor->DoExclusiveOperation(i->conf.processor_id, va, [&](u64 e) { return i->conf.callbacks->MemoryWriteExclusive64(va, v, e); }) ? 0 : 1; } + static void H_clrex(u32 c) { auto* i = I(c); if (i->conf.global_monitor) i->conf.global_monitor->ClearProcessor(i->conf.processor_id); } + + // Callbacks that may run guest-visible code (SVC, exceptions) get exact tick accounting around them. + template + static void WithTicks(Impl* i, F f) { + auto& s = i->st; + if (i->conf.enable_cycle_counting) { jit_js_stat(0, static_cast(s.cycles_to_run - s.ticks_left)); i->conf.callbacks->AddTicks(s.cycles_to_run - s.ticks_left); } + const double cb_t0 = jit_now_ms(); + f(); + jit_js_stat_ms(4, jit_now_ms() - cb_t0); + if (i->conf.enable_cycle_counting) { + s.cycles_to_run = i->conf.callbacks->GetTicksRemaining(); + s.ticks_left = s.cycles_to_run; + } + } + static void H_svc(u32 c, u32 imm) { auto* i = I(c); if (Crumbs() != 0) jit_js_crumb(3, i->st.regs[15], imm); if (std::getenv("JIT_TRACE")) std::fprintf(stderr, "[jit] svc %02x r0=%08x r1=%08x r2=%08x sp=%08x\n", imm, i->st.regs[0], i->st.regs[1], i->st.regs[2], i->st.regs[13]); WithTicks(i, [&] { i->conf.callbacks->CallSVC(imm); }); } + static void H_exc(u32 c, u32 pc, u32 ex) { auto* i = I(c); WithTicks(i, [&] { i->conf.callbacks->ExceptionRaised(pc, static_cast(ex)); }); } + static void H_interp(u32 c, u32 pc, u32 n) { auto* i = I(c); WithTicks(i, [&] { i->conf.callbacks->InterpreterFallback(pc, n); }); } + static void H_isb(u32 c) { I(c)->conf.callbacks->InstructionSynchronizationBarrierRaised(); } + static u32 H_getcpsr(u32 c) { return I(c)->st.Cpsr(); } + static void H_setcpsr(u32 c, u32 v) { I(c)->st.SetCpsr(v); } + static u32 H_getfpscr(u32 c) { return I(c)->st.Fpscr(); } + static void H_setfpscr(u32 c, u32 v) { I(c)->st.SetFpscr(v); } + static u32 H_cpget1(u32, u32 fn, u32 ua, u32 a0, u32 a1) { + return static_cast(reinterpret_cast(static_cast(fn))(reinterpret_cast(static_cast(ua)), a0, a1)); + } + static void H_cpcall(u32, u32 fn, u32 ua, u32 a0, u32 a1) { + reinterpret_cast(static_cast(fn))(reinterpret_cast(static_cast(ua)), a0, a1); + } + + // ----- VFP ----- + template + static auto Fp(u32 c, u32 fpcr, F f) { + auto* i = I(c); + if (hc_on) jit_js_crumb(20, i->st.regs[15], fpcr); // 20 = entering a soft-float helper + FP::FPSR sr{i->st.fpsr}; + auto r = f(FP::FPCR{fpcr}, sr); + if (hc_on) jit_js_crumb(21, i->st.regs[15], fpcr); // 21 = left it + i->st.fpsr = sr.Value(); + return r; + } +#define FP2(NAME, T, FN) static T NAME(u32 c, u32 fpcr, T a, T b) { return Fp(c, fpcr, [&](FP::FPCR cr, FP::FPSR& sr) { return Backend::Wasm::FPSoft::FN(a, b, cr, sr); }); } + FP2(H_fadd32, u32, Add) FP2(H_fsub32, u32, Sub) FP2(H_fmul32, u32, Mul) FP2(H_fdiv32, u32, Div) + FP2(H_fadd64, u64, Add) FP2(H_fsub64, u64, Sub) FP2(H_fmul64, u64, Mul) FP2(H_fdiv64, u64, Div) +#undef FP2 + static u32 H_fsqrt32(u32 c, u32 fpcr, u32 a) { return Fp(c, fpcr, [&](FP::FPCR cr, FP::FPSR& sr) { return Backend::Wasm::FPSoft::Sqrt(a, cr, sr); }); } + static u64 H_fsqrt64(u32 c, u32 fpcr, u64 a) { return Fp(c, fpcr, [&](FP::FPCR cr, FP::FPSR& sr) { return Backend::Wasm::FPSoft::Sqrt(a, cr, sr); }); } + // Packed halving and saturating arithmetic (ARM UHADD8, SHSUB16, UHASX, UQSUB8, QADD16 ...): kind numbers are fixed in emit_a32.cpp (PK). + static u32 H_pk(u32 kind, u32 a, u32 b) { + auto sat = [](s64 v, s64 lo, s64 hi) { return v < lo ? lo : (v > hi ? hi : v); }; + u32 r = 0; + if (kind >= 12) { // saturating, 8 or 16 bit lanes + const bool w16 = kind >= 16, sub = (kind & 2) != 0, sgn = (kind & 1) != 0; + const int bits = w16 ? 16 : 8, lanes = w16 ? 2 : 4; + const s64 lo = sgn ? -(s64(1) << (bits - 1)) : 0, hi = sgn ? (s64(1) << (bits - 1)) - 1 : (s64(1) << bits) - 1; + for (int i = 0; i < lanes; i++) { + const u32 m = (1u << bits) - 1; u32 x = (a >> (i * bits)) & m, y = (b >> (i * bits)) & m; + s64 xs = sgn ? static_cast(static_cast(x << (32 - bits)) >> (32 - bits)) : x, ys = sgn ? static_cast(static_cast(y << (32 - bits)) >> (32 - bits)) : y; + r |= (static_cast(sat(sub ? xs - ys : xs + ys, lo, hi)) & m) << (i * bits); + } + return r; + } + const bool sgn = (kind & 1) != 0; + auto lane = [&](u32 v, int shift, int bits) -> s64 { const u32 m = (1u << bits) - 1; u32 x = (v >> shift) & m; return sgn ? static_cast(static_cast(x << (32 - bits)) >> (32 - bits)) : static_cast(x); }; + if (kind < 8) { // halving add/sub, same lane + const bool w16 = kind >= 4, sub = (kind & 2) != 0; + const int bits = w16 ? 16 : 8, lanes = w16 ? 2 : 4; const u32 m = (1u << bits) - 1; + for (int i = 0; i < lanes; i++) { const s64 x = lane(a, i * bits, bits), y = lane(b, i * bits, bits); r |= (static_cast((sub ? x - y : x + y) >> 1) & m) << (i * bits); } + return r; + } + // halving exchanged 16-bit forms: kind 8,9 = add on the top half / subtract on the bottom half (UHASX), kind 10,11 = subtract on top / add on bottom (UHSAX) + const bool add_top = kind < 10; + const s64 hi = add_top ? lane(a, 16, 16) + lane(b, 0, 16) : lane(a, 16, 16) - lane(b, 0, 16); + const s64 lo = add_top ? lane(a, 0, 16) - lane(b, 16, 16) : lane(a, 0, 16) + lane(b, 16, 16); + return ((static_cast(hi >> 1) & 0xFFFF) << 16) | (static_cast(lo >> 1) & 0xFFFF); + } + static u32 H_fcmp32(u32 c, u32 fpcr, u32 a, u32 b, u32 exc) { return Fp(c, fpcr, [&](FP::FPCR cr, FP::FPSR& sr) { return Backend::Wasm::FPSoft::Compare(a, b, exc != 0, cr, sr); }); } + static u32 H_fcmp64(u32 c, u32 fpcr, u64 a, u64 b, u32 exc) { return Fp(c, fpcr, [&](FP::FPCR cr, FP::FPSR& sr) { return Backend::Wasm::FPSoft::Compare(a, b, exc != 0, cr, sr); }); } + static u64 H_fs2d(u32 c, u32 fpcr, u32 a, u32 rm) { return Fp(c, fpcr, [&](FP::FPCR cr, FP::FPSR& sr) { return Backend::Wasm::FPSoft::SingleToDouble(a, cr, static_cast(rm), sr); }); } + static u32 H_fd2s(u32 c, u32 fpcr, u64 a, u32 rm) { return Fp(c, fpcr, [&](FP::FPCR cr, FP::FPSR& sr) { return Backend::Wasm::FPSoft::DoubleToSingle(a, cr, static_cast(rm), sr); }); } + // mode: bit0 = 64-bit float, bit1 = unsigned, bits 2..7 = integer bits, bits 8..15 = fractional bits, bits 16..19 = rounding mode + static u32 H_ftoi(u32 c, u32 fpcr, u64 a, u32 mode) { + const bool f64 = mode & 1, uns = mode & 2; + const size_t ibits = (mode >> 2) & 63, fbits = (mode >> 8) & 255; + const auto rm = static_cast((mode >> 16) & 15); + return Fp(c, fpcr, [&](FP::FPCR cr, FP::FPSR& sr) { + return f64 ? Backend::Wasm::FPSoft::ToFixed(a, ibits, fbits, uns, cr, rm, sr) : Backend::Wasm::FPSoft::ToFixed(static_cast(a), ibits, fbits, uns, cr, rm, sr); + }); + } + static u64 H_itof(u32 c, u32 fpcr, u32 a, u32 mode) { + const bool f64 = mode & 1, uns = mode & 2; + const size_t ibits = (mode >> 2) & 63, fbits = (mode >> 8) & 255; + const auto rm = static_cast((mode >> 16) & 15); + return Fp(c, fpcr, [&](FP::FPCR cr, FP::FPSR& sr) -> u64 { + return f64 ? Backend::Wasm::FPSoft::FromFixed(a, ibits, fbits, uns, cr, rm, sr) : Backend::Wasm::FPSoft::FromFixed(a, ibits, fbits, uns, cr, rm, sr); + }); + } + + // ----- translation ----- + void InstallLink(u64 key, u32 slot) { + const u32 pc = static_cast(key), upper = static_cast(key >> 32); + link[(pc >> 1) & (kLinkEntries - 1)] = {pc, upper, slot, 0}; + } + + // Returns the function-table slot of the block at the current location, translating it if necessary; -1 on failure. + int GetBlock() { + const u64 key = st.GetLocationDescriptor().Value(); + if (auto it = cache.find(key); it != cache.end()) { + InstallLink(key, it->second); + return static_cast(it->second); + } + + const LocationDescriptor loc{IR::LocationDescriptor{key}}; + static const bool trace = std::getenv("JIT_TRACE") != nullptr; + if (trace) std::fprintf(stderr, "[jit] translate pc=%08x thumb=%d fpscr=%08x\n", loc.PC(), loc.TFlag() ? 1 : 0, loc.FPSCR().Value()); + if (Crumbs() != 0) jit_js_crumb(6, loc.PC(), 0); // 6 = about to translate + const double t_gen0 = jit_now_ms(); + IR::Block ir = Translate(loc, conf.callbacks, {conf.arch_version, conf.define_unpredictable_behaviour, conf.hook_hint_instructions}); + if (Crumbs() != 0) jit_js_crumb(7, loc.PC(), 0); // 7 = translated, optimizing + Optimization::A32GetSetElimination(ir, {}); + Optimization::DeadCodeElimination(ir); + Optimization::ConstantPropagation(ir); + Optimization::DeadCodeElimination(ir); + Optimization::IdentityRemovalPass(ir); + + Backend::Wasm::EmitOptions opt; + opt.link_base = static_cast(reinterpret_cast(link.data())); + opt.link_mask = kLinkEntries - 1; + // debugging switches (environment variables, or Module.ENV in the browser) + static const bool no_chain = std::getenv("JIT_NOCHAIN") != nullptr; + static const bool chain_tail = std::getenv("JIT_CHAIN_TAIL") != nullptr; // wasm tail calls (crash WebContent on iPhone, fast on desktop engines) + static const bool slow_fp = std::getenv("JIT_SLOW_FP") != nullptr; // force the soft-float helpers + static const bool no_fastmem = std::getenv("JIT_NOFASTMEM") != nullptr; // always call the memory helpers + opt.chain = !no_chain; + opt.chain_tail = chain_tail; + opt.fast_fp = !slow_fp; + opt.helper_fn = helper_fn; + { static const bool legacy = std::getenv("JIT_BATCH") && std::atoi(std::getenv("JIT_BATCH")) < 0; opt.legacy_layout = legacy; } // JIT_BATCH=-1: the old one-module-per-block layout + if (const char* m = std::getenv("JIT_FPMASK")) opt.fast_fp_mask = static_cast(std::strtoul(m, nullptr, 0)); // bit per kind: 1 add, 2 sub, 4 mul, 8 div, 16 sqrt, 32 cmp + UserConfig emit_conf = conf; + if (no_fastmem) { emit_conf.wasm_read_table = nullptr; emit_conf.wasm_write_table = nullptr; } + if (Crumbs() != 0) jit_js_crumb(8, loc.PC(), 0); // 8 = optimized, emitting wasm + auto res = Backend::Wasm::EmitA32Block(ir, emit_conf, opt); + jit_js_time(0, jit_now_ms() - t_gen0); + if (!res.ok()) { + char buf[64]; + std::snprintf(buf, sizeof buf, "block at 0x%08x: ", loc.PC()); + last_error = std::string(buf) + res.error; + std::fprintf(stderr, "[jit] %s\n", last_error.c_str()); + return -1; + } + const u32 slot = next_slot++; + { static const u32 hc_after = std::getenv("JIT_CRUMBH") ? static_cast(std::atoi(std::getenv("JIT_CRUMBH"))) : 0xFFFFFFFFu; if (slot >= hc_after) hc_on = true; } + if (Crumbs() != 0) jit_js_crumb(1, loc.PC(), slot); // 1 = compiled a block + if (const char* d = std::getenv("JIT_DUMP"); d && static_cast(std::strtoul(d, nullptr, 16)) == loc.PC()) { // JIT_DUMP=: write that block's module to jit-dump-.wasm + char fn[64]; std::snprintf(fn, sizeof fn, "jit-dump-%u.wasm", slot); + if (FILE* f = std::fopen(fn, "wb")) { std::fwrite(res.bytes.data(), 1, res.bytes.size(), f); std::fclose(f); } + std::fprintf(stderr, "[jit] dumped slot %u pc=%08x (%zu bytes) to %s\n", slot, loc.PC(), res.bytes.size(), fn); + } + static const int batch = std::getenv("JIT_BATCH") ? std::max(0, std::atoi(std::getenv("JIT_BATCH"))) : 256; // blocks packed into one WebAssembly instance (JIT_BATCH=0: off, -1: old one-module-per-block layout) + if (jit_js_compile(js_id, static_cast(slot), res.bytes.data(), static_cast(res.bytes.size()), static_cast(res.prefix_len), static_cast(res.code_off), static_cast(res.code_len), batch) < 0) { + last_error = "WebAssembly module rejected"; + return -1; + } + if (Crumbs() != 0) jit_js_crumb(4, loc.PC(), slot); // 4 = compile finished + cache[key] = slot; + { + const u32 start = loc.PC(), end = LocationDescriptor{ir.EndLocation()}.PC(); + if (slot_range.size() <= slot) slot_range.resize(slot + 1); + slot_range[slot] = {start, end > start ? end + 4 : start + 4096}; // +4: conservative (the last instruction and a thumb pair); a wrapped end falls back to a page + } + InstallLink(key, slot); + return static_cast(slot); + } + + void DropAllBlocks() { + const int dropped = static_cast(cache.size()); + cache.clear(); + std::fill(link.begin(), link.end(), Backend::Wasm::LinkEntry{0xFFFFFFFFu, 0, 0, 0}); + next_slot = 0; + slot_range.clear(); + pending_ranges.clear(); + jit_js_clear(js_id, dropped); + } + + // Forgets only the blocks that overlap guest [start, start+size): their table slots stay allocated but are no longer reachable from the + // lookup cache or the link cache, so the next visit translates the (possibly changed) code again. + void DropRange(u32 start, u32 size) { + const u64 end = static_cast(start) + size; + for (auto it = cache.begin(); it != cache.end();) { + const auto r = slot_range[it->second]; + if (static_cast(r.first) < end && r.second > start) it = cache.erase(it); else ++it; + } + std::fill(link.begin(), link.end(), Backend::Wasm::LinkEntry{0xFFFFFFFFu, 0, 0, 0}); // chained blocks find their next block through here: rebuilt from `cache` on demand + } + + void ApplyPendingRanges() { + for (const auto& [start, size] : pending_ranges) DropRange(start, size); + pending_ranges.clear(); + } +}; + +Jit::Jit(UserConfig conf) : impl(std::make_unique(std::move(conf))) {} +Jit::~Jit() = default; + +HaltReason Jit::Run() { + is_executing = true; + auto& s = impl->st; + auto* cb = impl->conf.callbacks; + const bool counting = impl->conf.enable_cycle_counting; + const u64 ticks = counting ? cb->GetTicksRemaining() : (u64(1) << 62); + s.cycles_to_run = ticks; + s.ticks_left = ticks; + s.chain_depth = 0; + int dispatches = 0; + const double run_t0 = jit_now_ms(); + + while (static_cast(s.ticks_left) > 0 && s.halt == 0) { + const u32 slots_before = impl->next_slot; + const int h = impl->GetBlock(); + if (Impl::Crumbs() != 0 && impl->next_slot != slots_before && h >= 0) jit_js_crumb(5, s.regs[15], static_cast(h)); // 5 = about to run a freshly compiled block + if (h < 0) { + s.halt |= static_cast(HaltReason::UserDefined8); + break; + } + s.chain_depth = 0; + if (const u32 ce = Impl::hc_on ? 1u : Impl::Crumbs(); ce != 0 && (++impl->crumb_n % ce) == 0) jit_js_crumb(2, s.regs[15], impl->crumb_n); // 2 = dispatch + ++dispatches; + jit_js_run(impl->js_id, h, static_cast(reinterpret_cast(&s))); + // EXPERIMENT, off by default: every compiled block is a live WebAssembly.Instance, and in WebKit the cost of each live instance is paid on every frame + // (a replay of a fixed frame: 59 Hz with 10 k instances, 43 Hz with 40 k, 25 Hz with 70 k). JIT_MAX_BLOCKS=N throws the whole cache away at N blocks. + // Measured: N=12000 thrashes the heavy scene (2 Hz), N=25000 gave 11.5 Hz (one run) against 28-31 Hz uncapped, so a flush-all cap is not a fix. + static const u32 kLiveLimit = std::getenv("JIT_MAX_BLOCKS") ? static_cast(std::strtoul(std::getenv("JIT_MAX_BLOCKS"), nullptr, 10)) : 0u; + if (kLiveLimit != 0 && impl->next_slot >= kLiveLimit && !impl->invalidate_pending) impl->DropAllBlocks(); + if (impl->invalidate_pending) { + impl->invalidate_pending = false; + impl->DropAllBlocks(); + s.halt &= ~static_cast(HaltReason::CacheInvalidation); + } else if (!impl->pending_ranges.empty()) { + impl->ApplyPendingRanges(); + s.halt &= ~static_cast(HaltReason::CacheInvalidation); + } + } + jit_js_stat_ms(3, jit_now_ms() - run_t0); + jit_js_stat(dispatches, counting ? static_cast(s.cycles_to_run - s.ticks_left) : 0.0); + if (counting) cb->AddTicks(s.cycles_to_run - s.ticks_left); + is_executing = false; + return static_cast(s.halt); +} + +HaltReason Jit::Step() { return HaltReason::Step; } // not implemented + +void Jit::ClearCache() { + impl->invalidate_pending = true; + if (!is_executing) { + impl->invalidate_pending = false; + impl->DropAllBlocks(); + } else { + impl->st.halt |= static_cast(HaltReason::CacheInvalidation); + } +} + +void Jit::InvalidateCacheRange(std::uint32_t start, std::size_t size) { + impl->pending_ranges.emplace_back(start, static_cast(size)); + if (!is_executing) { + impl->ApplyPendingRanges(); + } else { + impl->st.halt |= static_cast(HaltReason::CacheInvalidation); + } +} + +void Jit::Reset() { + const u32 ctx = impl->st.ctx; + impl->st = Backend::Wasm::A32JitState{}; + impl->st.ctx = ctx; +} + +void Jit::HaltExecution(HaltReason hr) { impl->st.halt |= static_cast(hr); } +void Jit::ClearHalt(HaltReason hr) { impl->st.halt &= ~static_cast(hr); } +std::array& Jit::Regs() { return impl->st.regs; } +const std::array& Jit::Regs() const { return impl->st.regs; } +std::array& Jit::ExtRegs() { return impl->st.ext_regs; } +const std::array& Jit::ExtRegs() const { return impl->st.ext_regs; } +std::uint32_t Jit::Cpsr() const { return impl->st.Cpsr(); } +void Jit::SetCpsr(std::uint32_t value) { impl->st.SetCpsr(value); } +std::uint32_t Jit::Fpscr() const { return impl->st.Fpscr(); } +void Jit::SetFpscr(std::uint32_t value) { impl->st.SetFpscr(value); } +void Jit::ClearExclusiveState() { impl->st.exclusive_state = 0; } +void Jit::DumpDisassembly() const {} +std::vector Jit::Disassemble() const { return {}; } + +} // namespace Dynarmic::A32 diff --git a/src/dynarmic/backend/wasm/a32_jitstate.cpp b/src/dynarmic/backend/wasm/a32_jitstate.cpp new file mode 100644 index 0000000..c924d99 --- /dev/null +++ b/src/dynarmic/backend/wasm/a32_jitstate.cpp @@ -0,0 +1,74 @@ +/* This file is part of the dynarmic project. + * Copyright (c) 2016 MerryMage + * SPDX-License-Identifier: 0BSD + */ + +#include "dynarmic/backend/wasm/a32_jitstate.h" + +#include +#include + +namespace Dynarmic::Backend::Wasm { + +u32 A32JitState::Cpsr() const { + u32 cpsr = 0; + + // NZCV flags + cpsr |= cpsr_nzcv; + // Q flag + cpsr |= cpsr_q; + // GE flags + cpsr |= mcl::bit::get_bit<31>(cpsr_ge) ? 1 << 19 : 0; + cpsr |= mcl::bit::get_bit<23>(cpsr_ge) ? 1 << 18 : 0; + cpsr |= mcl::bit::get_bit<15>(cpsr_ge) ? 1 << 17 : 0; + cpsr |= mcl::bit::get_bit<7>(cpsr_ge) ? 1 << 16 : 0; + // E flag, T flag + cpsr |= mcl::bit::get_bit<1>(upper_location_descriptor) ? 1 << 9 : 0; + cpsr |= mcl::bit::get_bit<0>(upper_location_descriptor) ? 1 << 5 : 0; + // IT state + cpsr |= static_cast(upper_location_descriptor & 0b11111100'00000000); + cpsr |= static_cast(upper_location_descriptor & 0b00000011'00000000) << 17; + // Other flags + cpsr |= cpsr_jaifm; + + return cpsr; +} + +void A32JitState::SetCpsr(u32 cpsr) { + // NZCV flags + cpsr_nzcv = cpsr & 0xF0000000; + // Q flag + cpsr_q = cpsr & (1 << 27); + // GE flags + cpsr_ge = 0; + cpsr_ge |= mcl::bit::get_bit<19>(cpsr) ? 0xFF000000 : 0; + cpsr_ge |= mcl::bit::get_bit<18>(cpsr) ? 0x00FF0000 : 0; + cpsr_ge |= mcl::bit::get_bit<17>(cpsr) ? 0x0000FF00 : 0; + cpsr_ge |= mcl::bit::get_bit<16>(cpsr) ? 0x000000FF : 0; + + upper_location_descriptor &= 0xFFFF0000; + // E flag, T flag + upper_location_descriptor |= mcl::bit::get_bit<9>(cpsr) ? 2 : 0; + upper_location_descriptor |= mcl::bit::get_bit<5>(cpsr) ? 1 : 0; + // IT state + upper_location_descriptor |= (cpsr >> 0) & 0b11111100'00000000; + upper_location_descriptor |= (cpsr >> 17) & 0b00000011'00000000; + + // Other flags + cpsr_jaifm = cpsr & 0x010001DF; +} + +constexpr u32 FPCR_MASK = A32::LocationDescriptor::FPSCR_MODE_MASK; +constexpr u32 FPSR_MASK = 0x0800'009f; + +u32 A32JitState::Fpscr() const { + return (upper_location_descriptor & 0xffff'0000) | fpsr | fpsr_nzcv; +} + +void A32JitState::SetFpscr(u32 fpscr) { + fpsr_nzcv = fpscr & 0xf000'0000; + fpsr = fpscr & FPSR_MASK; + upper_location_descriptor = (upper_location_descriptor & 0x0000'ffff) | (fpscr & FPCR_MASK); +} + +} // namespace Dynarmic::Backend::Wasm diff --git a/src/dynarmic/backend/wasm/a32_jitstate.h b/src/dynarmic/backend/wasm/a32_jitstate.h new file mode 100644 index 0000000..95cbcd9 --- /dev/null +++ b/src/dynarmic/backend/wasm/a32_jitstate.h @@ -0,0 +1,53 @@ +/* This file is part of the panda3ds-wasm project. + * SPDX-License-Identifier: GPL-3.0-or-later + * Layout and CPSR/FPSCR packing follow Dynarmic's arm64 backend (0BSD). + */ +#pragma once + +#include +#include + +#include + +#include "dynarmic/frontend/A32/a32_location_descriptor.h" +#include "dynarmic/ir/location_descriptor.h" + +namespace Dynarmic::Backend::Wasm { + +// Lives in WASM linear memory; generated blocks access it by byte offset (see offsetof uses in emit_a32.cpp). +struct A32JitState { + u32 cpsr_nzcv = 0; + u32 cpsr_q = 0; + u32 cpsr_jaifm = 0; + u32 cpsr_ge = 0; + + u32 fpsr = 0; + u32 fpsr_nzcv = 0; + + std::array regs{}; + + u32 upper_location_descriptor = 0; + + alignas(16) std::array ext_regs{}; + + u32 exclusive_state = 0; + + // WebAssembly backend additions + u32 halt = 0; ///< HaltReason bits; generated code returns to the dispatcher when non-zero at CheckHalt + u32 ctx = 0; ///< Jit::Impl* passed to host helper functions + alignas(8) u64 ticks_left = 0; + u64 cycles_to_run = 0; + u32 chain_depth = 0; ///< nesting depth of chained block calls (non-tail chaining mode) + + u32 Cpsr() const; + void SetCpsr(u32 cpsr); + + u32 Fpscr() const; + void SetFpscr(u32 fpscr); + + IR::LocationDescriptor GetLocationDescriptor() const { + return IR::LocationDescriptor{regs[15] | (static_cast(upper_location_descriptor) << 32)}; + } +}; + +} // namespace Dynarmic::Backend::Wasm diff --git a/src/dynarmic/backend/wasm/emit_a32.cpp b/src/dynarmic/backend/wasm/emit_a32.cpp new file mode 100644 index 0000000..e6b5ea0 --- /dev/null +++ b/src/dynarmic/backend/wasm/emit_a32.cpp @@ -0,0 +1,1632 @@ +/* This file is part of the panda3ds-wasm project. + * SPDX-License-Identifier: GPL-3.0-or-later + * + * IR -> WebAssembly translation for the A32 frontend. + * Semantics of the IR opcodes follow Dynarmic's other backends; unsupported opcodes make the block fail + * with an error naming the opcode (the caller reports it). + */ + +#include "dynarmic/backend/wasm/emit_a32.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "dynarmic/backend/wasm/a32_jitstate.h" +#include "dynarmic/backend/wasm/wasm_builder.h" +#include "dynarmic/frontend/A32/a32_location_descriptor.h" +#include "dynarmic/frontend/A32/a32_types.h" +#include "dynarmic/interface/A32/coprocessor.h" +#include "dynarmic/interface/halt_reason.h" +#include "dynarmic/ir/microinstruction.h" +#include "dynarmic/ir/opcodes.h" +#include "dynarmic/ir/terminal.h" + +namespace Dynarmic::Backend::Wasm { + +using IR::Opcode; + +namespace { + +constexpr u32 kOffNZCV = offsetof(A32JitState, cpsr_nzcv); +constexpr u32 kOffQ = offsetof(A32JitState, cpsr_q); +constexpr u32 kOffGE = offsetof(A32JitState, cpsr_ge); +constexpr u32 kOffFpsrNZCV = offsetof(A32JitState, fpsr_nzcv); +constexpr u32 kOffRegs = offsetof(A32JitState, regs); +constexpr u32 kOffUpper = offsetof(A32JitState, upper_location_descriptor); +constexpr u32 kOffExt = offsetof(A32JitState, ext_regs); +constexpr u32 kOffHalt = offsetof(A32JitState, halt); +constexpr u32 kOffCtx = offsetof(A32JitState, ctx); +constexpr u32 kOffTicks = offsetof(A32JitState, ticks_left); +constexpr u32 kOffDepth = offsetof(A32JitState, chain_depth); + +class Emitter { +public: + Emitter(IR::Block& b, const A32::UserConfig& cfg, const EmitOptions& o) : block(b), conf(cfg), opt(o) {} + + EmitResult Run() { + EmitResult res; + Generate(); + if (error.empty()) { res.bytes = opt.legacy_layout ? AssembleLegacy() : Assemble(); res.prefix_len = prefix_len; res.code_off = code_off; res.code_len = code_len; } + res.error = error; + return res; + } + +private: + IR::Block& block; + const A32::UserConfig& conf; + const EmitOptions opt; + bool uses_table = false; + Code c; + std::vector extra_locals; // types of locals after the single parameter (the state pointer) + std::unordered_map local_of; + std::vector import_order; // legacy layout only + std::map import_index; + int check_bit_local = -1; + std::string error; // first failure; emission stops once set + + void Fail(const std::string& msg) { if (error.empty()) error = msg; } + + // ---------- locals / values ---------- + ValType WT(IR::Type t) { + if (t == IR::Type::U64) return I64; + if (t == IR::Type::U1 || t == IR::Type::U8 || t == IR::Type::U16 || t == IR::Type::U32 || t == IR::Type::NZCVFlags) return I32; + Fail("unsupported IR value type " + std::to_string(static_cast(t))); + return I32; + } + // Scratch locals are recycled between IR instructions (a handler's temporaries are dead once it returns), which keeps big blocks + // from ending up with hundreds of locals. Locals that represent an IR value (Set / Alias) or are shared across instructions are + // never recycled. + std::vector free_locals[256]; // indexed by ValType byte + std::vector scratch_log; // temporaries handed out while emitting the current instruction + std::unordered_set pinned; + u32 FreshLocal(ValType t) { + extra_locals.push_back(t); + return static_cast(extra_locals.size()); // +1 for the parameter, -1 for zero-based + } + u32 Tmp(ValType t) { + u32 l; + if (!recycle_off && !free_locals[t].empty()) { l = free_locals[t].back(); free_locals[t].pop_back(); } + else l = FreshLocal(t); + scratch_log.push_back(l); + return l; + } + void RecycleScratch() { + for (u32 l : scratch_log) if (!pinned.count(l)) free_locals[extra_locals[l - 1]].push_back(l); + scratch_log.clear(); + } + const bool recycle_off = std::getenv("JIT_NORECYCLE") != nullptr; + // Pops the stack top into a fresh local that represents `inst`. + u32 Set(const IR::Inst* inst) { + const u32 l = FreshLocal(WT(inst->GetType())); + c.LocalSet(l); + local_of[inst] = l; + return l; + } + void Alias(const IR::Inst* inst, u32 local) { local_of[inst] = local; pinned.insert(local); } + void Ld(const IR::Value& v) { + if (v.IsImmediate()) { + switch (v.GetType()) { + case IR::Type::U1: c.I32Const(v.GetU1() ? 1 : 0); break; + case IR::Type::U8: c.I32Const(v.GetU8()); break; + case IR::Type::U16: c.I32Const(v.GetU16()); break; + case IR::Type::U32: c.I32Const(static_cast(v.GetU32())); break; + case IR::Type::U64: c.I64Const(static_cast(v.GetU64())); break; + default: Fail("unsupported immediate type"); c.I32Const(0); break; + } + return; + } + auto it = local_of.find(v.GetInst()); + if (it == local_of.end()) { + Fail("value used before definition (producer opcode " + IR::GetNameOf(v.GetInst()->GetOpcode()) + ")"); + c.I32Const(0); + return; + } + c.LocalGet(it->second); + } + void LdL(u32 local) { c.LocalGet(local); } + + // ---------- helpers ---------- + // Every module imports ALL helpers in kHelpers order, so a helper's function index is its position there. This makes the function bodies of + // different blocks interchangeable: the JS side can put many of them into one module (one WebAssembly.Instance for a whole batch of blocks). + u32 Import(const char* name) { + if (opt.legacy_layout) { // old layout: only the helpers this block uses, numbered in order of first use (kept to compare against in measurements) + auto it = import_index.find(name); + if (it != import_index.end()) return it->second; + const u32 idx = static_cast(import_order.size()); + import_order.push_back(name); + import_index[name] = idx; + return idx; + } + for (u32 i = 0; i < std::size(kHelpers); i++) if (std::strcmp(kHelpers[i].name, name) == 0) return i; + Fail(std::string("unknown helper ") + name); + return 0; + } + void PushCtx() { + c.LocalGet(0); + c.I32Load(kOffCtx); + } + // Helpers are the host's own C++ functions (H_r32, ...) living in the main module's function table, so a block calls them with call_indirect through that table + // (imported as table 1) instead of importing them as functions: every imported function makes JavaScriptCore generate a wasm-to-JS stub in its executable + // memory pool (a block with 50 imports ran that pool dry: crash in ExecutableAllocator::allocate, on a Mac after 12 k blocks), and calls through the table need none. + void CallHelper(const char* name) { + if (opt.legacy_layout || !opt.helper_fn) { c.Call(Import(name)); return; } + const u32 i = Import(name); + c.I32Const(static_cast(opt.helper_fn[i])); + c.CallIndirect(i + 1, 1); // type i + 1 = signature of kHelpers[i]; table 1 = the host's function table + } + + // ---------- state ---------- + void StateLoad32(u32 off) { + c.LocalGet(0); + c.I32Load(off); + } + void SubTicks(u64 n) { + if (!conf.enable_cycle_counting || n == 0) return; + c.LocalGet(0); + c.LocalGet(0); + c.I64Load(kOffTicks); + c.I64Const(static_cast(n)); + c.I64Sub(); + c.I64Store(kOffTicks); + } + + // Leaves 0/1 on the stack for condition `cond` evaluated on the current CPSR flags. + void EmitCond(IR::Cond cond) { + auto flag = [&](int bit) { // 0/1 + StateLoad32(kOffNZCV); + c.I32Const(bit); + c.I32ShrU(); + c.I32Const(1); + c.I32And(); + }; + switch (cond) { + case IR::Cond::EQ: flag(30); break; + case IR::Cond::NE: flag(30); c.I32Eqz(); break; + case IR::Cond::CS: flag(29); break; + case IR::Cond::CC: flag(29); c.I32Eqz(); break; + case IR::Cond::MI: flag(31); break; + case IR::Cond::PL: flag(31); c.I32Eqz(); break; + case IR::Cond::VS: flag(28); break; + case IR::Cond::VC: flag(28); c.I32Eqz(); break; + case IR::Cond::HI: flag(29); flag(30); c.I32Eqz(); c.I32And(); break; + case IR::Cond::LS: flag(29); c.I32Eqz(); flag(30); c.I32Or(); break; + case IR::Cond::GE: flag(31); flag(28); c.I32Eq(); break; + case IR::Cond::LT: flag(31); flag(28); c.I32Ne(); break; + case IR::Cond::GT: flag(30); c.I32Eqz(); flag(31); flag(28); c.I32Eq(); c.I32And(); break; + case IR::Cond::LE: flag(30); flag(31); flag(28); c.I32Ne(); c.I32Or(); break; + case IR::Cond::AL: + case IR::Cond::NV: c.I32Const(1); break; + default: Fail("bad condition"); c.I32Const(1); break; + } + } + + // ---------- terminals ---------- + static u32 UpperOf(IR::LocationDescriptor d) { + return static_cast(A32::LocationDescriptor{d}.SetSingleStepping(false).UniqueHash() >> 32); + } + void SetUpper(IR::LocationDescriptor new_loc, IR::LocationDescriptor old_loc) { + const u32 nu = UpperOf(new_loc); + if (nu == UpperOf(old_loc)) return; + c.LocalGet(0); + c.I32Const(static_cast(nu)); + c.I32Store(kOffUpper); + } + void Ret(int code) { + c.I32Const(code); + c.Return(); + } + void EmitTerm(const IR::Terminal& t, IR::LocationDescriptor initial) { + if (auto* p = boost::get(&t)) { EmitLink(p->next, initial); } + else if (auto* p2 = boost::get(&t)) { EmitLink(p2->next, initial); } + else if (boost::get(&t) || boost::get(&t) || boost::get(&t)) { ChainDynamic(); Ret(0); } + else if (auto* i = boost::get(&t)) { + PushCtx(); + c.I32Const(static_cast(A32::LocationDescriptor{i->next}.PC())); + c.I32Const(static_cast(i->num_instructions)); + CallHelper("interp"); + Ret(1); + } else if (auto* f = boost::get(&t)) { + EmitCond(f->if_); + c.If(); + EmitTerm(f->then_, initial); + c.Else(); + EmitTerm(f->else_, initial); + c.End(); + } else if (auto* cb = boost::get(&t)) { + if (check_bit_local < 0) { Fail("CheckBit terminal without SetCheckBit"); return; } + c.LocalGet(static_cast(check_bit_local)); + c.If(); + EmitTerm(cb->then_, initial); + c.Else(); + EmitTerm(cb->else_, initial); + c.End(); + } else if (auto* ch = boost::get(&t)) { + StateLoad32(kOffHalt); + c.If(); + Ret(1); + c.End(); + EmitTerm(ch->else_, initial); + } else { + Fail("unsupported terminal"); + } + } + void EmitLink(IR::LocationDescriptor next, IR::LocationDescriptor initial) { + SetUpper(next, initial); + c.LocalGet(0); + c.I32Const(static_cast(A32::LocationDescriptor{next}.PC())); + c.I32Store(kOffRegs + 4 * 15); + ChainStatic(A32::LocationDescriptor{next}.PC(), UpperOf(next)); + Ret(0); + } + + // ----- block chaining (tail calls through the shared function table) ----- + // Leaves nothing on the stack; falls through when the chain cannot continue (cache miss, out of ticks, halt requested). + void EmitChainGuard() { // pushes 1 if we may continue running guest code without returning to the host loop + c.LocalGet(0); c.I64Load(kOffTicks); c.I64Const(0); c.I64GtS(); + StateLoad32(kOffHalt); c.I32Eqz(); + c.I32And(); + } + void EmitChainLookup(u32 entry_local, u32 pc_local, u32 up_local) { + // if (entry.pc == pc && entry.upper == upper) run table[entry.slot](state) next + c.LocalGet(entry_local); c.I32Load(0); c.LocalGet(pc_local); c.I32Eq(); + c.LocalGet(entry_local); c.I32Load(4); c.LocalGet(up_local); c.I32Eq(); + c.I32And(); + if (opt.chain_tail) { + c.If(); + c.LocalGet(0); + c.LocalGet(entry_local); c.I32Load(8); + c.ReturnCallIndirect(0, 0); + c.End(); + } else { + // Nested call with a bounded depth: the callee chains further; when the limit is hit everything unwinds back to the host loop. + c.LocalGet(0); c.I32Load(kOffDepth); c.I32Const(static_cast(opt.chain_depth_limit)); c.I32LtU(); + c.I32And(); + c.If(); + c.LocalGet(0); c.LocalGet(0); c.I32Load(kOffDepth); c.I32Const(1); c.I32Add(); c.I32Store(kOffDepth); + c.LocalGet(0); + c.LocalGet(entry_local); c.I32Load(8); + c.CallIndirect(0, 0); + c.LocalGet(0); c.LocalGet(0); c.I32Load(kOffDepth); c.I32Const(1); c.I32Sub(); c.I32Store(kOffDepth); + c.Return(); + c.End(); + } + } + void ChainStatic(u32 pc, u32 upper) { + if (!opt.chain) return; + uses_table = true; + const u32 idx = (pc >> 1) & opt.link_mask; + const u32 entry = Tmp(I32), pcl = Tmp(I32), upl = Tmp(I32); + EmitChainGuard(); + c.If(); + c.I32Const(static_cast(opt.link_base + idx * sizeof(LinkEntry))); c.LocalSet(entry); + c.I32Const(static_cast(pc)); c.LocalSet(pcl); + c.I32Const(static_cast(upper)); c.LocalSet(upl); + EmitChainLookup(entry, pcl, upl); + c.End(); + } + void ChainDynamic() { + if (!opt.chain) return; + uses_table = true; + const u32 entry = Tmp(I32), pcl = Tmp(I32), upl = Tmp(I32); + EmitChainGuard(); + c.If(); + StateLoad32(kOffRegs + 4 * 15); c.LocalSet(pcl); + StateLoad32(kOffUpper); c.LocalSet(upl); + c.LocalGet(pcl); c.I32Const(1); c.I32ShrU(); c.I32Const(static_cast(opt.link_mask)); c.I32And(); + c.I32Const(4); c.I32Shl(); c.I32Const(static_cast(opt.link_base)); c.I32Add(); c.LocalSet(entry); + EmitChainLookup(entry, pcl, upl); + c.End(); + } + + // ---------- main ---------- + void Generate() { + const A32::LocationDescriptor location{block.Location()}; + const IR::LocationDescriptor initial = location.SetSingleStepping(false); + + if (block.GetCondition() != IR::Cond::AL) { + EmitCond(block.GetCondition()); + c.I32Eqz(); + c.If(); + SubTicks(block.ConditionFailedCycleCount()); + EmitTerm(IR::Term::LinkBlock{block.ConditionFailedLocation()}, initial); + c.End(); + } + SubTicks(block.CycleCount()); + + for (auto& inst : block) { + EmitInst(&inst); + RecycleScratch(); + if (!error.empty()) return; + } + + EmitTerm(block.GetTerminal(), initial); + c.Unreachable(); + } + + IR::Inst* P(IR::Inst* inst, Opcode op) { return inst->GetAssociatedPseudoOperation(op); } + u32 Fpcr() const { return A32::LocationDescriptor{block.Location()}.FPSCR().Value(); } + + // N and Z in bits 31 and 30 for the 32-bit value in `local`. + void EmitNZ(u32 local) { + c.LocalGet(local); + c.I32Const(static_cast(0x80000000u)); + c.I32And(); + c.LocalGet(local); + c.I32Eqz(); + c.I32Const(30); + c.I32Shl(); + c.I32Or(); + } + + void EmitNZpseudo(IR::Inst*, u32) {} // GetNZFromOp is emitted by the pseudo-instruction itself (see EmitInst) + + // ---------- shifts ---------- + // Emits res into a new local and, if requested, the carry. `which`: 0 LSL, 1 LSR, 2 ASR, 3 ROR. + void EmitShift32(IR::Inst* inst, int which) { + auto* carry_inst = P(inst, Opcode::GetCarryFromOp); + const IR::Value v = inst->GetArg(0), amt = inst->GetArg(1), cin = inst->GetArg(2); + const u32 res = Tmp(I32); + u32 car = 0; + if (carry_inst) car = Tmp(I32); + + if (amt.IsImmediate()) { + const u32 n = amt.GetU8(); + if (n == 0) { + Ld(v); c.LocalSet(res); + if (carry_inst) { Ld(cin); c.LocalSet(car); } + } else if (which == 0) { // LSL + if (n < 32) { + Ld(v); c.I32Const(n); c.I32Shl(); c.LocalSet(res); + if (carry_inst) { Ld(v); c.I32Const(32 - n); c.I32ShrU(); c.I32Const(1); c.I32And(); c.LocalSet(car); } + } else { + c.I32Const(0); c.LocalSet(res); + if (carry_inst) { if (n == 32) { Ld(v); c.I32Const(1); c.I32And(); } else { c.I32Const(0); } c.LocalSet(car); } + } + } else if (which == 1) { // LSR + if (n < 32) { + Ld(v); c.I32Const(n); c.I32ShrU(); c.LocalSet(res); + if (carry_inst) { Ld(v); c.I32Const(n - 1); c.I32ShrU(); c.I32Const(1); c.I32And(); c.LocalSet(car); } + } else { + c.I32Const(0); c.LocalSet(res); + if (carry_inst) { if (n == 32) { Ld(v); c.I32Const(31); c.I32ShrU(); } else { c.I32Const(0); } c.LocalSet(car); } + } + } else if (which == 2) { // ASR + const u32 m = n < 32 ? n : 31; + Ld(v); c.I32Const(m); c.I32ShrS(); c.LocalSet(res); + if (carry_inst) { Ld(v); c.I32Const(n < 32 ? n - 1 : 31); c.I32ShrS(); c.I32Const(1); c.I32And(); c.LocalSet(car); } + } else { // ROR + Ld(v); c.I32Const(n & 31); c.I32Rotr(); c.LocalSet(res); + if (carry_inst) { c.LocalGet(res); c.I32Const(31); c.I32ShrU(); c.LocalSet(car); } + } + } else { + // amount known only at run time + if (which == 0) { + Ld(v); Ld(amt); c.I32Shl(); c.I32Const(0); Ld(amt); c.I32Const(32); c.I32LtU(); c.Select(); c.LocalSet(res); + } else if (which == 1) { + Ld(v); Ld(amt); c.I32ShrU(); c.I32Const(0); Ld(amt); c.I32Const(32); c.I32LtU(); c.Select(); c.LocalSet(res); + } else if (which == 2) { + Ld(v); Ld(amt); c.I32Const(31); Ld(amt); c.I32Const(32); c.I32LtU(); c.Select(); c.I32ShrS(); c.LocalSet(res); + } else { + Ld(v); Ld(amt); c.I32Rotr(); c.LocalSet(res); + } + if (carry_inst) { + Ld(amt); c.I32Eqz(); + c.If(I32); + Ld(cin); + c.Else(); + if (which == 0) { + Ld(amt); c.I32Const(32); c.I32LeU(); + c.If(I32); Ld(v); c.I32Const(32); Ld(amt); c.I32Sub(); c.I32ShrU(); c.I32Const(1); c.I32And(); c.Else(); c.I32Const(0); c.End(); + } else if (which == 1) { + Ld(amt); c.I32Const(32); c.I32LeU(); + c.If(I32); Ld(v); Ld(amt); c.I32Const(1); c.I32Sub(); c.I32ShrU(); c.I32Const(1); c.I32And(); c.Else(); c.I32Const(0); c.End(); + } else if (which == 2) { + Ld(v); Ld(amt); c.I32Const(1); c.I32Sub(); c.I32Const(31); Ld(amt); c.I32Const(32); c.I32LtU(); c.Select(); c.I32ShrS(); c.I32Const(1); c.I32And(); + } else { + c.LocalGet(res); c.I32Const(31); c.I32ShrU(); + } + c.End(); + c.LocalSet(car); + } + } + Alias(inst, res); + if (carry_inst) Alias(carry_inst, car); + EmitNZpseudo(inst, res); + } + + // ---------- packed SIMD operations (ARMv6) ---------- + void EmitPackedAddSub8(IR::Inst* inst, bool sub, bool signed_ge) { + const u32 al = Tmp(I32); Ld(inst->GetArg(0)); c.LocalSet(al); + const u32 bl = Tmp(I32); Ld(inst->GetArg(1)); c.LocalSet(bl); + + if (!sub) { + c.LocalGet(al); c.I32Const(0x7f7f7f7f); c.I32And(); + c.LocalGet(bl); c.I32Const(0x7f7f7f7f); c.I32And(); + c.I32Add(); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); + c.I32Const(static_cast(0x80808080u)); c.I32And(); + c.I32Xor(); + } else { + c.LocalGet(al); c.I32Const(static_cast(0x80808080u)); c.I32Or(); + c.LocalGet(bl); c.I32Const(0x7f7f7f7f); c.I32And(); + c.I32Sub(); + c.LocalGet(al); c.LocalGet(bl); c.I32Const(-1); c.I32Xor(); c.I32Xor(); + c.I32Const(static_cast(0x80808080u)); c.I32And(); + c.I32Xor(); + } + const u32 res = Tmp(I32); + c.LocalSet(res); + Alias(inst, res); + + if (auto* ge = P(inst, Opcode::GetGEFromOp)) { + if (!sub && !signed_ge) { + c.LocalGet(al); c.LocalGet(bl); c.I32And(); + c.LocalGet(al); c.LocalGet(bl); c.I32Or(); + c.LocalGet(res); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } else if (!sub && signed_ge) { + c.LocalGet(al); c.I32Const(-1); c.I32Xor(); + c.LocalGet(bl); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); + c.LocalGet(res); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } else if (sub && !signed_ge) { + const u32 not_b = Tmp(I32); + c.LocalGet(bl); c.I32Const(-1); c.I32Xor(); c.LocalSet(not_b); + c.LocalGet(al); c.LocalGet(not_b); c.I32And(); + c.LocalGet(al); c.LocalGet(not_b); c.I32Or(); + c.LocalGet(res); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } else { + c.LocalGet(al); c.I32Const(-1); c.I32Xor(); c.LocalGet(bl); c.I32And(); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); c.I32Const(-1); c.I32Xor(); + c.LocalGet(res); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } + c.I32Const(static_cast(0x80808080u)); c.I32And(); + const u32 gel = Tmp(I32); + c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(1); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(2); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(4); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + Alias(ge, gel); + } + } + + void EmitPackedAddSub16(IR::Inst* inst, bool sub, bool signed_ge) { + const u32 al = Tmp(I32); Ld(inst->GetArg(0)); c.LocalSet(al); + const u32 bl = Tmp(I32); Ld(inst->GetArg(1)); c.LocalSet(bl); + + if (!sub) { + c.LocalGet(al); c.I32Const(0x7fff7fff); c.I32And(); + c.LocalGet(bl); c.I32Const(0x7fff7fff); c.I32And(); + c.I32Add(); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); + c.I32Const(static_cast(0x80008000u)); c.I32And(); + c.I32Xor(); + } else { + c.LocalGet(al); c.I32Const(static_cast(0x80008000u)); c.I32Or(); + c.LocalGet(bl); c.I32Const(0x7fff7fff); c.I32And(); + c.I32Sub(); + c.LocalGet(al); c.LocalGet(bl); c.I32Const(-1); c.I32Xor(); c.I32Xor(); + c.I32Const(static_cast(0x80008000u)); c.I32And(); + c.I32Xor(); + } + const u32 res = Tmp(I32); + c.LocalSet(res); + Alias(inst, res); + + if (auto* ge = P(inst, Opcode::GetGEFromOp)) { + if (!sub && !signed_ge) { + c.LocalGet(al); c.LocalGet(bl); c.I32And(); + c.LocalGet(al); c.LocalGet(bl); c.I32Or(); + c.LocalGet(res); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } else if (!sub && signed_ge) { + c.LocalGet(al); c.I32Const(-1); c.I32Xor(); + c.LocalGet(bl); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); + c.LocalGet(res); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } else if (sub && !signed_ge) { + const u32 not_b = Tmp(I32); + c.LocalGet(bl); c.I32Const(-1); c.I32Xor(); c.LocalSet(not_b); + c.LocalGet(al); c.LocalGet(not_b); c.I32And(); + c.LocalGet(al); c.LocalGet(not_b); c.I32Or(); + c.LocalGet(res); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } else { + c.LocalGet(al); c.I32Const(-1); c.I32Xor(); c.LocalGet(bl); c.I32And(); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); c.I32Const(-1); c.I32Xor(); + c.LocalGet(res); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } + c.I32Const(static_cast(0x80008000u)); c.I32And(); + const u32 gel = Tmp(I32); + c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(1); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(2); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(4); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(8); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + Alias(ge, gel); + } + } + + void EmitPackedMixed16(IR::Inst* inst, bool add_top, bool signed_ge) { + const u32 al = Tmp(I32); Ld(inst->GetArg(0)); c.LocalSet(al); + const u32 bl = Tmp(I32); Ld(inst->GetArg(1)); + c.I32Const(16); c.I32Rotr(); c.LocalSet(bl); + + const u32 sum = Tmp(I32); + c.LocalGet(al); c.I32Const(0x7fff7fff); c.I32And(); + c.LocalGet(bl); c.I32Const(0x7fff7fff); c.I32And(); + c.I32Add(); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); + c.I32Const(static_cast(0x80008000u)); c.I32And(); + c.I32Xor(); + c.LocalSet(sum); + + const u32 diff = Tmp(I32); + c.LocalGet(al); c.I32Const(static_cast(0x80008000u)); c.I32Or(); + c.LocalGet(bl); c.I32Const(0x7fff7fff); c.I32And(); + c.I32Sub(); + c.LocalGet(al); c.LocalGet(bl); c.I32Const(-1); c.I32Xor(); c.I32Xor(); + c.I32Const(static_cast(0x80008000u)); c.I32And(); + c.I32Xor(); + c.LocalSet(diff); + + const u32 top_val = add_top ? sum : diff; + const u32 bot_val = add_top ? diff : sum; + c.LocalGet(top_val); c.I32Const(static_cast(0xffff0000u)); c.I32And(); + c.LocalGet(bot_val); c.I32Const(0x0000ffff); c.I32And(); + c.I32Or(); + const u32 res = Tmp(I32); + c.LocalSet(res); + Alias(inst, res); + + if (auto* ge = P(inst, Opcode::GetGEFromOp)) { + const u32 ge_add = Tmp(I32); + if (!signed_ge) { + c.LocalGet(al); c.LocalGet(bl); c.I32And(); + c.LocalGet(al); c.LocalGet(bl); c.I32Or(); + c.LocalGet(sum); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } else { + c.LocalGet(al); c.I32Const(-1); c.I32Xor(); + c.LocalGet(bl); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); + c.LocalGet(sum); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } + c.I32Const(static_cast(0x80008000u)); c.I32And(); + c.LocalSet(ge_add); + + const u32 ge_sub = Tmp(I32); + if (!signed_ge) { + const u32 not_b = Tmp(I32); + c.LocalGet(bl); c.I32Const(-1); c.I32Xor(); c.LocalSet(not_b); + c.LocalGet(al); c.LocalGet(not_b); c.I32And(); + c.LocalGet(al); c.LocalGet(not_b); c.I32Or(); + c.LocalGet(diff); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } else { + c.LocalGet(al); c.I32Const(-1); c.I32Xor(); c.LocalGet(bl); c.I32And(); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); c.I32Const(-1); c.I32Xor(); + c.LocalGet(diff); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + } + c.I32Const(static_cast(0x80008000u)); c.I32And(); + c.LocalSet(ge_sub); + + const u32 top_ge = add_top ? ge_add : ge_sub; + const u32 bot_ge = add_top ? ge_sub : ge_add; + c.LocalGet(top_ge); c.I32Const(static_cast(0x80000000u)); c.I32And(); + c.LocalGet(bot_ge); c.I32Const(0x00008000); c.I32And(); + c.I32Or(); + const u32 gel = Tmp(I32); + c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(1); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(2); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(4); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + c.LocalGet(gel); c.LocalGet(gel); c.I32Const(8); c.I32ShrU(); c.I32Or(); c.LocalSet(gel); + Alias(ge, gel); + } + } + + void EmitPackedAbsDiffSum8(IR::Inst* inst) { + const u32 al = Tmp(I32); Ld(inst->GetArg(0)); c.LocalSet(al); + const u32 bl = Tmp(I32); Ld(inst->GetArg(1)); c.LocalSet(bl); + const u32 acc = Tmp(I32); + c.I32Const(0); c.LocalSet(acc); + for (int i = 0; i < 4; i++) { + const u32 da = Tmp(I32); + const u32 db = Tmp(I32); + c.LocalGet(al); if (i) { c.I32Const(i * 8); c.I32ShrU(); } c.I32Const(0xff); c.I32And(); c.LocalSet(da); + c.LocalGet(bl); if (i) { c.I32Const(i * 8); c.I32ShrU(); } c.I32Const(0xff); c.I32And(); c.LocalSet(db); + c.LocalGet(acc); + c.LocalGet(da); c.LocalGet(db); c.I32Sub(); + c.LocalGet(db); c.LocalGet(da); c.I32Sub(); + c.LocalGet(da); c.LocalGet(db); c.I32GeU(); + c.Select(); + c.I32Add(); + c.LocalSet(acc); + } + Alias(inst, acc); + } + + // ---------- per-instruction ---------- + void Bin32(IR::Inst* inst, void (Code::*op)()) { + Ld(inst->GetArg(0)); + Ld(inst->GetArg(1)); + (c.*op)(); + const u32 r = Set(inst); + EmitNZpseudo(inst, r); + } + + void EmitInst(IR::Inst* inst) { + const Opcode op = inst->GetOpcode(); + switch (op) { + // ----- pseudo operations: produced together with their parent instruction ----- + case Opcode::GetNZFromOp: { + const IR::Value src = inst->GetArg(0); + if (src.IsImmediate()) { // constant-folded parent + const u32 v = src.GetType() == IR::Type::U64 ? static_cast(src.GetU64()) : src.GetU32(); + c.I32Const(static_cast((v & 0x80000000u) | (v == 0 ? 0x40000000u : 0u))); + Set(inst); + return; + } + auto it = local_of.find(src.GetInst()); + if (it == local_of.end()) { Fail("GetNZFromOp: parent not emitted"); return; } + EmitNZ(it->second); + Set(inst); + return; + } + case Opcode::PushRSB: // return-stack-buffer hint, not needed here + case Opcode::GetCarryFromOp: + case Opcode::GetOverflowFromOp: + case Opcode::GetNZCVFromOp: + case Opcode::GetGEFromOp: + case Opcode::Void: + case Opcode::Breakpoint: + case Opcode::A32DataSynchronizationBarrier: + case Opcode::A32DataMemoryBarrier: + if (local_of.count(inst) == 0 && (op == Opcode::GetCarryFromOp || op == Opcode::GetOverflowFromOp || op == Opcode::GetNZCVFromOp || op == Opcode::GetNZFromOp || op == Opcode::GetGEFromOp)) { + // Only an error if somebody uses it, which Ld() reports; unused pseudo-ops are removed by DCE. + } + return; + case Opcode::A32InstructionSynchronizationBarrier: + if (conf.hook_isb) { PushCtx(); CallHelper("isb"); } + return; + + // ----- A32 state ----- + case Opcode::A32GetRegister: + StateLoad32(kOffRegs + 4 * static_cast(inst->GetArg(0).GetA32RegRef())); + Set(inst); + return; + case Opcode::A32SetRegister: + c.LocalGet(0); + Ld(inst->GetArg(1)); + c.I32Store(kOffRegs + 4 * static_cast(inst->GetArg(0).GetA32RegRef())); + return; + case Opcode::A32GetExtendedRegister32: { + const auto reg = inst->GetArg(0).GetA32ExtRegRef(); + c.LocalGet(0); + c.I32Load(kOffExt + 4 * static_cast(static_cast(reg) - static_cast(A32::ExtReg::S0))); + Set(inst); + return; + } + case Opcode::A32GetExtendedRegister64: { + const auto reg = inst->GetArg(0).GetA32ExtRegRef(); + c.LocalGet(0); + c.I64Load(kOffExt + 8 * static_cast(static_cast(reg) - static_cast(A32::ExtReg::D0))); + Set(inst); + return; + } + case Opcode::A32SetExtendedRegister32: { + const auto reg = inst->GetArg(0).GetA32ExtRegRef(); + c.LocalGet(0); + Ld(inst->GetArg(1)); + c.I32Store(kOffExt + 4 * static_cast(static_cast(reg) - static_cast(A32::ExtReg::S0))); + return; + } + case Opcode::A32SetExtendedRegister64: { + const auto reg = inst->GetArg(0).GetA32ExtRegRef(); + c.LocalGet(0); + Ld(inst->GetArg(1)); + c.I64Store(kOffExt + 8 * static_cast(static_cast(reg) - static_cast(A32::ExtReg::D0))); + return; + } + case Opcode::A32GetCpsr: PushCtx(); CallHelper("getcpsr"); Set(inst); return; + case Opcode::A32SetCpsr: PushCtx(); Ld(inst->GetArg(0)); CallHelper("setcpsr"); return; + case Opcode::A32GetFpscr: PushCtx(); CallHelper("getfpscr"); Set(inst); return; + case Opcode::A32SetFpscr: PushCtx(); Ld(inst->GetArg(0)); CallHelper("setfpscr"); return; + case Opcode::A32GetFpscrNZCV: StateLoad32(kOffFpsrNZCV); Set(inst); return; + case Opcode::A32SetFpscrNZCV: c.LocalGet(0); Ld(inst->GetArg(0)); c.I32Store(kOffFpsrNZCV); return; + case Opcode::A32SetCpsrNZCV: + case Opcode::A32SetCpsrNZCVRaw: + c.LocalGet(0); Ld(inst->GetArg(0)); c.I32Store(kOffNZCV); return; + case Opcode::A32SetCpsrNZCVQ: + c.LocalGet(0); Ld(inst->GetArg(0)); c.I32Const(static_cast(0xf0000000u)); c.I32And(); c.I32Store(kOffNZCV); + c.LocalGet(0); Ld(inst->GetArg(0)); c.I32Const(0x08000000); c.I32And(); c.I32Store(kOffQ); + return; + case Opcode::A32SetCpsrNZ: + c.LocalGet(0); + StateLoad32(kOffNZCV); c.I32Const(0x30000000); c.I32And(); + Ld(inst->GetArg(0)); c.I32Or(); + c.I32Store(kOffNZCV); + return; + case Opcode::A32SetCpsrNZC: + c.LocalGet(0); + StateLoad32(kOffNZCV); c.I32Const(0x10000000); c.I32And(); + // An immediate here is the "NZ not needed" marker from GetSetElimination: contributes nothing. + if (!inst->GetArg(0).IsImmediate()) { Ld(inst->GetArg(0)); c.I32Or(); } + Ld(inst->GetArg(1)); c.I32Const(29); c.I32Shl(); c.I32Or(); + c.I32Store(kOffNZCV); + return; + case Opcode::A32GetCFlag: + StateLoad32(kOffNZCV); c.I32Const(29); c.I32ShrU(); c.I32Const(1); c.I32And(); Set(inst); + return; + case Opcode::A32OrQFlag: + c.LocalGet(0); + StateLoad32(kOffQ); Ld(inst->GetArg(0)); c.I32Const(27); c.I32Shl(); c.I32Or(); + c.I32Store(kOffQ); + return; + case Opcode::A32GetGEFlags: StateLoad32(kOffGE); Set(inst); return; + case Opcode::A32SetGEFlags: c.LocalGet(0); Ld(inst->GetArg(0)); c.I32Store(kOffGE); return; + case Opcode::A32SetGEFlagsCompressed: + // ge = ((x >> 16) * 0x00204081) & 0x01010101; ge = (ge << 8) - ge + c.LocalGet(0); + Ld(inst->GetArg(0)); c.I32Const(16); c.I32ShrU(); c.I32Const(0x00204081); c.I32Mul(); c.I32Const(0x01010101); c.I32And(); + c.I32Const(8); c.I32Shl(); + Ld(inst->GetArg(0)); c.I32Const(16); c.I32ShrU(); c.I32Const(0x00204081); c.I32Mul(); c.I32Const(0x01010101); c.I32And(); + c.I32Sub(); + c.I32Store(kOffGE); + return; + case Opcode::A32SetCheckBit: + if (check_bit_local < 0) check_bit_local = static_cast(FreshLocal(I32)); + Ld(inst->GetArg(0)); + c.LocalSet(static_cast(check_bit_local)); + return; + case Opcode::A32BXWritePC: { + const u32 upper_without_t = UpperOf(block.EndLocation()) & 0xFFFFFFFE; + const IR::Value pc = inst->GetArg(0); + if (pc.IsImmediate()) { + const u32 new_pc = pc.GetU32(); + const bool t = mcl::bit::get_bit<0>(new_pc); + c.LocalGet(0); c.I32Const(static_cast(new_pc & (t ? 0xFFFFFFFEu : 0xFFFFFFFCu))); c.I32Store(kOffRegs + 4 * 15); + c.LocalGet(0); c.I32Const(static_cast(upper_without_t | (t ? 1u : 0u))); c.I32Store(kOffUpper); + } else { + const u32 pcl = Tmp(I32); + Ld(pc); c.LocalSet(pcl); + // regs[15] = pc & ~(t ? 1 : 3) == pc & (0xFFFFFFFC | (t << 1)) + c.LocalGet(0); + c.LocalGet(pcl); c.I32Const(static_cast(0xFFFFFFFCu)); c.LocalGet(pcl); c.I32Const(1); c.I32And(); c.I32Const(1); c.I32Shl(); c.I32Or(); c.I32And(); + c.I32Store(kOffRegs + 4 * 15); + c.LocalGet(0); + c.I32Const(static_cast(upper_without_t)); c.LocalGet(pcl); c.I32Const(1); c.I32And(); c.I32Or(); + c.I32Store(kOffUpper); + } + return; + } + case Opcode::A32UpdateUpperLocationDescriptor: + for (auto& i : block) if (i.GetOpcode() == Opcode::A32BXWritePC) return; + { + const u32 nu = UpperOf(block.EndLocation()); + if (nu != UpperOf(block.Location())) { + c.LocalGet(0); c.I32Const(static_cast(nu)); c.I32Store(kOffUpper); + } + } + return; + case Opcode::A32CallSupervisor: + PushCtx(); c.I32Const(static_cast(inst->GetArg(0).GetU32())); CallHelper("svc"); + return; + case Opcode::A32ExceptionRaised: + PushCtx(); c.I32Const(static_cast(inst->GetArg(0).GetU32())); c.I32Const(static_cast(inst->GetArg(1).GetU64())); CallHelper("exc"); + return; + case Opcode::A32ClearExclusive: PushCtx(); CallHelper("clrex"); return; + + // ----- memory ----- + case Opcode::A32ReadMemory8: EmitFastRead(inst, 1, "r8"); return; + case Opcode::A32ReadMemory16: EmitFastRead(inst, 2, "r16"); return; + case Opcode::A32ReadMemory32: EmitFastRead(inst, 4, "r32"); return; + case Opcode::A32ReadMemory64: EmitFastRead(inst, 8, "r64"); return; + case Opcode::A32ExclusiveReadMemory8: case Opcode::A32ExclusiveReadMemory16: case Opcode::A32ExclusiveReadMemory32: case Opcode::A32ExclusiveReadMemory64: { + static const char* names[] = {"xr8", "xr16", "xr32", "xr64"}; + const int k = op == Opcode::A32ExclusiveReadMemory8 ? 0 : op == Opcode::A32ExclusiveReadMemory16 ? 1 : op == Opcode::A32ExclusiveReadMemory32 ? 2 : 3; + PushCtx(); Ld(inst->GetArg(1)); CallHelper(names[k]); + if (k == 0) { c.I32Const(0xff); c.I32And(); } + if (k == 1) { c.I32Const(0xffff); c.I32And(); } + Set(inst); + return; + } + case Opcode::A32WriteMemory8: EmitFastWrite(inst, 1, "w8"); return; + case Opcode::A32WriteMemory16: EmitFastWrite(inst, 2, "w16"); return; + case Opcode::A32WriteMemory32: EmitFastWrite(inst, 4, "w32"); return; + case Opcode::A32WriteMemory64: EmitFastWrite(inst, 8, "w64"); return; + case Opcode::A32ExclusiveWriteMemory8: case Opcode::A32ExclusiveWriteMemory16: case Opcode::A32ExclusiveWriteMemory32: case Opcode::A32ExclusiveWriteMemory64: { + static const char* names[] = {"xw8", "xw16", "xw32", "xw64"}; + const int k = op == Opcode::A32ExclusiveWriteMemory8 ? 0 : op == Opcode::A32ExclusiveWriteMemory16 ? 1 : op == Opcode::A32ExclusiveWriteMemory32 ? 2 : 3; + PushCtx(); Ld(inst->GetArg(1)); Ld(inst->GetArg(2)); CallHelper(names[k]); Set(inst); + return; + } + + // ----- coprocessor (CP15 on the 3DS) ----- + case Opcode::A32CoprocGetOneWord: case Opcode::A32CoprocGetTwoWords: case Opcode::A32CoprocSendOneWord: + case Opcode::A32CoprocSendTwoWords: case Opcode::A32CoprocInternalOperation: + EmitCoproc(inst); + return; + + // ----- integer / bit operations ----- + case Opcode::Identity: Ld(inst->GetArg(0)); Set(inst); return; + case Opcode::LeastSignificantWord: Ld(inst->GetArg(0)); c.I32WrapI64(); Set(inst); return; + case Opcode::LeastSignificantHalf: Ld(inst->GetArg(0)); c.I32Const(0xffff); c.I32And(); Set(inst); return; + case Opcode::LeastSignificantByte: Ld(inst->GetArg(0)); c.I32Const(0xff); c.I32And(); Set(inst); return; + case Opcode::MostSignificantWord: { + Ld(inst->GetArg(0)); + const u32 v = Tmp(I64); + c.LocalSet(v); + if (auto* carry = P(inst, Opcode::GetCarryFromOp)) { + c.LocalGet(v); c.I64Const(31); c.I64ShrU(); c.I32WrapI64(); c.I32Const(1); c.I32And(); Set(carry); + } + c.LocalGet(v); c.I64Const(32); c.I64ShrU(); c.I32WrapI64(); Set(inst); + return; + } + case Opcode::GetCFlagFromNZCV: Ld(inst->GetArg(0)); c.I32Const(29); c.I32ShrU(); c.I32Const(1); c.I32And(); Set(inst); return; + case Opcode::MostSignificantBit: Ld(inst->GetArg(0)); c.I32Const(31); c.I32ShrU(); Set(inst); return; + case Opcode::IsZero32: Ld(inst->GetArg(0)); c.I32Eqz(); Set(inst); return; + case Opcode::IsZero64: Ld(inst->GetArg(0)); c.I64Eqz(); Set(inst); return; + case Opcode::TestBit: Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); c.I64ExtendI32U(); c.I64ShrU(); c.I32WrapI64(); c.I32Const(1); c.I32And(); Set(inst); return; + case Opcode::ConditionalSelect32: + Ld(inst->GetArg(1)); Ld(inst->GetArg(2)); EmitCond(inst->GetArg(0).GetCond()); c.Select(); Set(inst); + return; + case Opcode::ConditionalSelect64: + Ld(inst->GetArg(1)); Ld(inst->GetArg(2)); EmitCond(inst->GetArg(0).GetCond()); c.Select(); Set(inst); + return; + case Opcode::Pack2x32To1x64: + Ld(inst->GetArg(0)); c.I64ExtendI32U(); Ld(inst->GetArg(1)); c.I64ExtendI32U(); c.I64Const(32); c.I64Shl(); c.I64Or(); Set(inst); + return; + case Opcode::LogicalShiftLeft32: EmitShift32(inst, 0); return; + case Opcode::LogicalShiftRight32: EmitShift32(inst, 1); return; + case Opcode::ArithmeticShiftRight32: EmitShift32(inst, 2); return; + case Opcode::RotateRight32: EmitShift32(inst, 3); return; + case Opcode::RotateRightExtended: { + auto* carry = P(inst, Opcode::GetCarryFromOp); + if (carry) { Ld(inst->GetArg(0)); c.I32Const(1); c.I32And(); Set(carry); } + Ld(inst->GetArg(1)); c.I32Const(31); c.I32Shl(); Ld(inst->GetArg(0)); c.I32Const(1); c.I32ShrU(); c.I32Or(); + const u32 r = Set(inst); + EmitNZpseudo(inst, r); + return; + } + case Opcode::Add32: case Opcode::Sub32: { + const bool sub = op == Opcode::Sub32; + auto* carry = P(inst, Opcode::GetCarryFromOp); + auto* ov = P(inst, Opcode::GetOverflowFromOp); + auto* nzcv = P(inst, Opcode::GetNZCVFromOp); + const IR::Value a = inst->GetArg(0), b = inst->GetArg(1), cin = inst->GetArg(2); + const u32 sum = Tmp(I64), res = Tmp(I32); + Ld(a); c.I64ExtendI32U(); + Ld(b); if (sub) { c.I32Const(-1); c.I32Xor(); } c.I64ExtendI32U(); c.I64Add(); + Ld(cin); c.I64ExtendI32U(); c.I64Add(); + c.LocalSet(sum); + c.LocalGet(sum); c.I32WrapI64(); c.LocalSet(res); + Alias(inst, res); + u32 cl = 0, ol = 0; + if (carry || nzcv) { cl = Tmp(I32); c.LocalGet(sum); c.I64Const(32); c.I64ShrU(); c.I32WrapI64(); c.LocalSet(cl); } + if (ov || nzcv) { + ol = Tmp(I32); + if (!sub) { Ld(a); c.LocalGet(res); c.I32Xor(); Ld(b); c.LocalGet(res); c.I32Xor(); c.I32And(); } + else { Ld(a); Ld(b); c.I32Xor(); Ld(a); c.LocalGet(res); c.I32Xor(); c.I32And(); } + c.I32Const(31); c.I32ShrU(); c.LocalSet(ol); + } + if (carry) Alias(carry, cl); + if (ov) Alias(ov, ol); + if (nzcv) { + EmitNZ(res); + c.LocalGet(cl); c.I32Const(29); c.I32Shl(); c.I32Or(); + c.LocalGet(ol); c.I32Const(28); c.I32Shl(); c.I32Or(); + Set(nzcv); + } + EmitNZpseudo(inst, res); + return; + } + case Opcode::Add64: case Opcode::Sub64: { + Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); + if (op == Opcode::Add64) { c.I64Add(); Ld(inst->GetArg(2)); c.I64ExtendI32U(); c.I64Add(); } + else { c.I64Sub(); Ld(inst->GetArg(2)); c.I64ExtendI32U(); c.I64Add(); c.I64Const(1); c.I64Sub(); } + Set(inst); + return; + } + case Opcode::Mul32: Bin32(inst, &Code::I32Mul); return; + case Opcode::Mul64: Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); c.I64Mul(); Set(inst); return; + case Opcode::And32: Bin32(inst, &Code::I32And); return; + case Opcode::Eor32: Bin32(inst, &Code::I32Xor); return; + case Opcode::Or32: Bin32(inst, &Code::I32Or); return; + case Opcode::AndNot32: + Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); c.I32Const(-1); c.I32Xor(); c.I32And(); + { const u32 r = Set(inst); EmitNZpseudo(inst, r); } + return; + case Opcode::Not32: + Ld(inst->GetArg(0)); c.I32Const(-1); c.I32Xor(); + { const u32 r = Set(inst); EmitNZpseudo(inst, r); } + return; + case Opcode::And64: Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); c.I64And(); Set(inst); return; + case Opcode::Eor64: Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); c.I64Xor(); Set(inst); return; + case Opcode::Or64: Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); c.I64Or(); Set(inst); return; + case Opcode::SignExtendByteToWord: Ld(inst->GetArg(0)); c.I32Extend8S(); Set(inst); return; + case Opcode::SignExtendHalfToWord: Ld(inst->GetArg(0)); c.I32Extend16S(); Set(inst); return; + case Opcode::ZeroExtendByteToWord: case Opcode::ZeroExtendHalfToWord: Ld(inst->GetArg(0)); Set(inst); return; + case Opcode::SignExtendWordToLong: Ld(inst->GetArg(0)); c.I64ExtendI32S(); Set(inst); return; + case Opcode::ZeroExtendWordToLong: case Opcode::ZeroExtendByteToLong: case Opcode::ZeroExtendHalfToLong: + Ld(inst->GetArg(0)); c.I64ExtendI32U(); Set(inst); return; + case Opcode::CountLeadingZeros32: Ld(inst->GetArg(0)); c.I32Clz(); Set(inst); return; + case Opcode::ByteReverseWord: { + const u32 x = Tmp(I32); + Ld(inst->GetArg(0)); c.LocalSet(x); + c.LocalGet(x); c.I32Const(24); c.I32Shl(); + c.LocalGet(x); c.I32Const(0xff00); c.I32And(); c.I32Const(8); c.I32Shl(); c.I32Or(); + c.LocalGet(x); c.I32Const(8); c.I32ShrU(); c.I32Const(0xff00); c.I32And(); c.I32Or(); + c.LocalGet(x); c.I32Const(24); c.I32ShrU(); c.I32Or(); + Set(inst); + return; + } + case Opcode::ByteReverseHalf: { + const u32 x = Tmp(I32); + Ld(inst->GetArg(0)); c.LocalSet(x); + c.LocalGet(x); c.I32Const(0xff); c.I32And(); c.I32Const(8); c.I32Shl(); + c.LocalGet(x); c.I32Const(8); c.I32ShrU(); c.I32Or(); + Set(inst); + return; + } + case Opcode::ReplicateBit32: + c.I32Const(0); Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); c.I32ShrU(); c.I32Const(1); c.I32And(); c.I32Sub(); Set(inst); + return; + case Opcode::UnsignedDiv32: case Opcode::SignedDiv32: { + const bool sg = op == Opcode::SignedDiv32; + const u32 b = Tmp(I32); + Ld(inst->GetArg(1)); c.LocalSet(b); + c.LocalGet(b); c.I32Eqz(); + c.If(I32); + c.I32Const(0); + c.Else(); + if (sg) { + // INT_MIN / -1 overflows in wasm (traps); ARM defines the result as INT_MIN + Ld(inst->GetArg(0)); c.I32Const(static_cast(0x80000000u)); c.I32Eq(); c.LocalGet(b); c.I32Const(-1); c.I32Eq(); c.I32And(); + c.If(I32); c.I32Const(static_cast(0x80000000u)); c.Else(); Ld(inst->GetArg(0)); c.LocalGet(b); c.I32DivS(); c.End(); + } else { + Ld(inst->GetArg(0)); c.LocalGet(b); c.I32DivU(); + } + c.End(); + Set(inst); + return; + } + + // ----- packed operations (ARMv6 SIMD) ----- + case Opcode::PackedAddU8: EmitPackedAddSub8(inst, false, false); return; + case Opcode::PackedAddS8: EmitPackedAddSub8(inst, false, true); return; + case Opcode::PackedSubU8: EmitPackedAddSub8(inst, true, false); return; + case Opcode::PackedSubS8: EmitPackedAddSub8(inst, true, true); return; + case Opcode::PackedAddU16: EmitPackedAddSub16(inst, false, false); return; + case Opcode::PackedAddS16: EmitPackedAddSub16(inst, false, true); return; + case Opcode::PackedSubU16: EmitPackedAddSub16(inst, true, false); return; + case Opcode::PackedSubS16: EmitPackedAddSub16(inst, true, true); return; + case Opcode::PackedAddSubU16: EmitPackedMixed16(inst, true, false); return; + case Opcode::PackedAddSubS16: EmitPackedMixed16(inst, true, true); return; + case Opcode::PackedSubAddU16: EmitPackedMixed16(inst, false, false); return; + case Opcode::PackedSubAddS16: EmitPackedMixed16(inst, false, true); return; + case Opcode::PackedSelect: { + const u32 ge = Tmp(I32); Ld(inst->GetArg(0)); c.LocalSet(ge); + Ld(inst->GetArg(2)); c.LocalGet(ge); c.I32And(); + Ld(inst->GetArg(1)); c.LocalGet(ge); c.I32Const(-1); c.I32Xor(); c.I32And(); + c.I32Or(); + Set(inst); + return; + } + case Opcode::PackedAbsDiffSumU8: EmitPackedAbsDiffSum8(inst); return; +#define PK(op, id) case Opcode::op: c.I32Const(id); Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); CallHelper("pk"); Set(inst); return; + PK(PackedHalvingAddU8, 0) PK(PackedHalvingAddS8, 1) PK(PackedHalvingSubU8, 2) PK(PackedHalvingSubS8, 3) + PK(PackedHalvingAddU16, 4) PK(PackedHalvingAddS16, 5) PK(PackedHalvingSubU16, 6) PK(PackedHalvingSubS16, 7) + PK(PackedHalvingAddSubU16, 8) PK(PackedHalvingAddSubS16, 9) PK(PackedHalvingSubAddU16, 10) PK(PackedHalvingSubAddS16, 11) + PK(PackedSaturatedAddU8, 12) PK(PackedSaturatedAddS8, 13) PK(PackedSaturatedSubU8, 14) PK(PackedSaturatedSubS8, 15) + PK(PackedSaturatedAddU16, 16) PK(PackedSaturatedAddS16, 17) PK(PackedSaturatedSubU16, 18) PK(PackedSaturatedSubS16, 19) +#undef PK + + // ----- saturation ----- + case Opcode::SignedSaturatedAddWithFlag32: { + const u32 al = Tmp(I32); Ld(inst->GetArg(0)); c.LocalSet(al); + const u32 bl = Tmp(I32); Ld(inst->GetArg(1)); c.LocalSet(bl); + const u32 sum = Tmp(I32); + c.LocalGet(al); c.LocalGet(bl); c.I32Add(); c.LocalSet(sum); + const u32 ovl = Tmp(I32); + c.LocalGet(sum); c.LocalGet(al); c.I32Xor(); + c.LocalGet(sum); c.LocalGet(bl); c.I32Xor(); + c.I32And(); c.I32Const(31); c.I32ShrU(); c.LocalSet(ovl); + + // saturated value: (sum >> 31) ^ 0x80000000 (INT_MAX if sum < 0, INT_MIN if sum >= 0) + c.LocalGet(sum); c.I32Const(31); c.I32ShrS(); c.I32Const(static_cast(0x80000000u)); c.I32Xor(); + c.LocalGet(sum); + c.LocalGet(ovl); + c.Select(); + const u32 res = Tmp(I32); + c.LocalSet(res); + Alias(inst, res); + if (auto* ov = P(inst, Opcode::GetOverflowFromOp)) { + Alias(ov, ovl); + } + return; + } + case Opcode::SignedSaturatedSubWithFlag32: { + const u32 al = Tmp(I32); Ld(inst->GetArg(0)); c.LocalSet(al); + const u32 bl = Tmp(I32); Ld(inst->GetArg(1)); c.LocalSet(bl); + const u32 diff = Tmp(I32); + c.LocalGet(al); c.LocalGet(bl); c.I32Sub(); c.LocalSet(diff); + const u32 ovl = Tmp(I32); + c.LocalGet(al); c.LocalGet(bl); c.I32Xor(); + c.LocalGet(al); c.LocalGet(diff); c.I32Xor(); + c.I32And(); c.I32Const(31); c.I32ShrU(); c.LocalSet(ovl); + + // saturated value: (diff >> 31) ^ 0x80000000 + c.LocalGet(diff); c.I32Const(31); c.I32ShrS(); c.I32Const(static_cast(0x80000000u)); c.I32Xor(); + c.LocalGet(diff); + c.LocalGet(ovl); + c.Select(); + const u32 res = Tmp(I32); + c.LocalSet(res); + Alias(inst, res); + if (auto* ov = P(inst, Opcode::GetOverflowFromOp)) { + Alias(ov, ovl); + } + return; + } + case Opcode::SignedSaturation: { + const size_t N = inst->GetArg(1).GetU8(); + const u32 vl = Tmp(I32); + Ld(inst->GetArg(0)); + c.LocalSet(vl); + if (N == 32) { + Alias(inst, vl); + if (auto* ov = P(inst, Opcode::GetOverflowFromOp)) { + const u32 ovl = Tmp(I32); + c.I32Const(0); + c.LocalSet(ovl); + Alias(ov, ovl); + } + return; + } + const u32 max_val = (1u << (N - 1)) - 1; + const s32 min_val = static_cast(~u32{0} << (N - 1)); + + c.I32Const(static_cast(max_val)); + c.I32Const(min_val); + c.LocalGet(vl); + c.LocalGet(vl); + c.I32Const(min_val); + c.I32LtS(); + c.Select(); + c.LocalGet(vl); + c.I32Const(static_cast(max_val)); + c.I32GtS(); + c.Select(); + const u32 res = Tmp(I32); + c.LocalSet(res); + Alias(inst, res); + + if (auto* ov = P(inst, Opcode::GetOverflowFromOp)) { + const u32 ovl = Tmp(I32); + c.LocalGet(res); + c.LocalGet(vl); + c.I32Ne(); + c.LocalSet(ovl); + Alias(ov, ovl); + } + return; + } + case Opcode::UnsignedSaturation: { + const size_t N = inst->GetArg(1).GetU8(); + const u32 saturated_value = (1u << N) - 1; + const u32 vl = Tmp(I32); + Ld(inst->GetArg(0)); + c.LocalSet(vl); + + c.I32Const(0); + c.I32Const(static_cast(saturated_value)); + c.LocalGet(vl); + c.LocalGet(vl); + c.I32Const(static_cast(saturated_value)); + c.I32GtS(); + c.Select(); + c.LocalGet(vl); + c.I32Const(0); + c.I32LtS(); + c.Select(); + const u32 res = Tmp(I32); + c.LocalSet(res); + Alias(inst, res); + + if (auto* ov = P(inst, Opcode::GetOverflowFromOp)) { + const u32 ovl = Tmp(I32); + c.LocalGet(vl); + c.I32Const(static_cast(saturated_value)); + c.I32GtU(); + c.LocalSet(ovl); + Alias(ov, ovl); + } + return; + } + + // ----- VFP scalar floating point (bit-exact soft-float helpers; FPCR is known per block) ----- + case Opcode::FPAdd32: if (FastFp32(inst, 0)) return; goto fp_generic; + case Opcode::FPSub32: if (FastFp32(inst, 1)) return; goto fp_generic; + case Opcode::FPMul32: if (FastFp32(inst, 2)) return; goto fp_generic; + case Opcode::FPDiv32: if (FastFp32(inst, 3)) return; goto fp_generic; + case Opcode::FPSqrt32: if (FastFp32(inst, 4)) return; goto fp_generic; + fp_generic: + case Opcode::FPAdd64: case Opcode::FPSub64: case Opcode::FPMul64: case Opcode::FPDiv64: { + if (op == Opcode::FPSqrt32) { + PushCtx(); c.I32Const(static_cast(Fpcr())); Ld(inst->GetArg(0)); CallHelper("fsqrt32"); Set(inst); + return; + } + const bool is64 = op == Opcode::FPAdd64 || op == Opcode::FPSub64 || op == Opcode::FPMul64 || op == Opcode::FPDiv64; + static const char* n32[] = {"fadd32", "fsub32", "fmul32", "fdiv32"}; + static const char* n64[] = {"fadd64", "fsub64", "fmul64", "fdiv64"}; + const int k = (op == Opcode::FPAdd32 || op == Opcode::FPAdd64) ? 0 : (op == Opcode::FPSub32 || op == Opcode::FPSub64) ? 1 + : (op == Opcode::FPMul32 || op == Opcode::FPMul64) ? 2 : 3; + PushCtx(); c.I32Const(static_cast(Fpcr())); Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); + CallHelper(is64 ? n64[k] : n32[k]); Set(inst); + return; + } + case Opcode::FPSqrt64: + PushCtx(); c.I32Const(static_cast(Fpcr())); Ld(inst->GetArg(0)); + CallHelper("fsqrt64"); Set(inst); + return; + case Opcode::FPAbs32: Ld(inst->GetArg(0)); c.I32Const(0x7fffffff); c.I32And(); Set(inst); return; + case Opcode::FPNeg32: Ld(inst->GetArg(0)); c.I32Const(static_cast(0x80000000u)); c.I32Xor(); Set(inst); return; + case Opcode::FPAbs64: Ld(inst->GetArg(0)); c.I64Const(0x7fffffffffffffffLL); c.I64And(); Set(inst); return; + case Opcode::FPNeg64: Ld(inst->GetArg(0)); c.I64Const(static_cast(0x8000000000000000ULL)); c.I64Xor(); Set(inst); return; + case Opcode::FPCompare32: case Opcode::FPCompare64: + if (op == Opcode::FPCompare32 && FastCompare32(inst)) return; + PushCtx(); c.I32Const(static_cast(Fpcr())); Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); Ld(inst->GetArg(2)); + CallHelper(op == Opcode::FPCompare32 ? "fcmp32" : "fcmp64"); Set(inst); + return; + case Opcode::FPSingleToDouble: + PushCtx(); c.I32Const(static_cast(Fpcr())); Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); CallHelper("fs2d"); Set(inst); + return; + case Opcode::FPDoubleToSingle: + PushCtx(); c.I32Const(static_cast(Fpcr())); Ld(inst->GetArg(0)); Ld(inst->GetArg(1)); CallHelper("fd2s"); Set(inst); + return; + case Opcode::FPSingleToFixedS32: case Opcode::FPSingleToFixedU32: case Opcode::FPSingleToFixedS16: case Opcode::FPSingleToFixedU16: + case Opcode::FPDoubleToFixedS32: case Opcode::FPDoubleToFixedU32: case Opcode::FPDoubleToFixedS16: case Opcode::FPDoubleToFixedU16: { + const bool src64 = op == Opcode::FPDoubleToFixedS32 || op == Opcode::FPDoubleToFixedU32 || op == Opcode::FPDoubleToFixedS16 || op == Opcode::FPDoubleToFixedU16; + const bool uns = op == Opcode::FPSingleToFixedU32 || op == Opcode::FPSingleToFixedU16 || op == Opcode::FPDoubleToFixedU32 || op == Opcode::FPDoubleToFixedU16; + const u32 ibits = (op == Opcode::FPSingleToFixedS16 || op == Opcode::FPSingleToFixedU16 || op == Opcode::FPDoubleToFixedS16 || op == Opcode::FPDoubleToFixedU16) ? 16 : 32; + const u32 mode = (src64 ? 1u : 0u) | (uns ? 2u : 0u) | (ibits << 2) | (static_cast(inst->GetArg(1).GetU8()) << 8) | (static_cast(inst->GetArg(2).GetU8()) << 16); + PushCtx(); c.I32Const(static_cast(Fpcr())); Ld(inst->GetArg(0)); if (!src64) c.I64ExtendI32U(); c.I32Const(static_cast(mode)); + CallHelper("ftoi"); Set(inst); + return; + } + case Opcode::FPFixedS32ToSingle: case Opcode::FPFixedU32ToSingle: case Opcode::FPFixedS16ToSingle: case Opcode::FPFixedU16ToSingle: + case Opcode::FPFixedS32ToDouble: case Opcode::FPFixedU32ToDouble: case Opcode::FPFixedS16ToDouble: case Opcode::FPFixedU16ToDouble: { + const bool dst64 = op == Opcode::FPFixedS32ToDouble || op == Opcode::FPFixedU32ToDouble || op == Opcode::FPFixedS16ToDouble || op == Opcode::FPFixedU16ToDouble; + const bool uns = op == Opcode::FPFixedU32ToSingle || op == Opcode::FPFixedU16ToSingle || op == Opcode::FPFixedU32ToDouble || op == Opcode::FPFixedU16ToDouble; + const u32 ibits = (op == Opcode::FPFixedS16ToSingle || op == Opcode::FPFixedU16ToSingle || op == Opcode::FPFixedS16ToDouble || op == Opcode::FPFixedU16ToDouble) ? 16 : 32; + const u32 mode = (dst64 ? 1u : 0u) | (uns ? 2u : 0u) | (ibits << 2) | (static_cast(inst->GetArg(1).GetU8()) << 8) | (static_cast(inst->GetArg(2).GetU8()) << 16); + PushCtx(); c.I32Const(static_cast(Fpcr())); Ld(inst->GetArg(0)); c.I32Const(static_cast(mode)); + CallHelper("itof"); + if (!dst64) c.I32WrapI64(); + Set(inst); + return; + } + + default: + Fail("unsupported IR opcode " + IR::GetNameOf(op)); + return; + } + } + + // Inline page-table access (same tables Panda3DS uses): aligned, non-page-crossing accesses to mapped pages go straight to + // host memory; everything else (unaligned, MMIO / unmapped) calls the host helper. + void EmitFastRead(IR::Inst* inst, int size, const char* slow) { + const bool wide = size == 8; + if (!conf.wasm_read_table) { + PushCtx(); Ld(inst->GetArg(1)); CallHelper(slow); + if (size == 1) { c.I32Const(0xff); c.I32And(); } + if (size == 2) { c.I32Const(0xffff); c.I32And(); } + Set(inst); + return; + } + const u32 v = Tmp(I32), pte = Tmp(I32); + Ld(inst->GetArg(1)); c.LocalSet(v); + c.Block(wide ? I64 : I32); // $done + c.Block(); // $slow + if (size > 1) { c.LocalGet(v); c.I32Const(size == 2 ? 1 : 3); c.I32And(); c.BrIf(0); } + if (wide) { c.LocalGet(v); c.I32Const(0xfff); c.I32And(); c.I32Const(0xff8); c.I32GtU(); c.BrIf(0); } + c.I32Const(static_cast(reinterpret_cast(conf.wasm_read_table))); + c.LocalGet(v); c.I32Const(12); c.I32ShrU(); c.I32Const(2); c.I32Shl(); c.I32Add(); c.I32Load(0); + c.LocalTee(pte); c.I32Eqz(); c.BrIf(0); + c.LocalGet(pte); c.LocalGet(v); c.I32Const(0xfff); c.I32And(); c.I32Add(); + if (size == 1) c.I32Load8U(0); else if (size == 2) c.I32Load16U(0); else if (size == 4) c.I32Load(0); else c.I64Load(0); + c.Br(1); + c.End(); // $slow + PushCtx(); c.LocalGet(v); CallHelper(slow); + if (size == 1) { c.I32Const(0xff); c.I32And(); } + if (size == 2) { c.I32Const(0xffff); c.I32And(); } + c.End(); // $done + Set(inst); + } + void EmitFastWrite(IR::Inst* inst, int size, const char* slow) { + const bool wide = size == 8; + if (!conf.wasm_write_table) { + PushCtx(); Ld(inst->GetArg(1)); Ld(inst->GetArg(2)); CallHelper(slow); + return; + } + const u32 v = Tmp(I32), pte = Tmp(I32), val = Tmp(wide ? I64 : I32); + Ld(inst->GetArg(1)); c.LocalSet(v); + Ld(inst->GetArg(2)); c.LocalSet(val); + c.Block(); // $done + c.Block(); // $slow + if (size > 1) { c.LocalGet(v); c.I32Const(size == 2 ? 1 : 3); c.I32And(); c.BrIf(0); } + if (wide) { c.LocalGet(v); c.I32Const(0xfff); c.I32And(); c.I32Const(0xff8); c.I32GtU(); c.BrIf(0); } + c.I32Const(static_cast(reinterpret_cast(conf.wasm_write_table))); + c.LocalGet(v); c.I32Const(12); c.I32ShrU(); c.I32Const(2); c.I32Shl(); c.I32Add(); c.I32Load(0); + c.LocalTee(pte); c.I32Eqz(); c.BrIf(0); + c.LocalGet(pte); c.LocalGet(v); c.I32Const(0xfff); c.I32And(); c.I32Add(); + c.LocalGet(val); + if (size == 1) c.I32Store8(0); else if (size == 2) c.I32Store16(0); else if (size == 4) c.I32Store(0); else c.I64Store(0); + c.Br(1); + c.End(); // $slow + PushCtx(); c.LocalGet(v); c.LocalGet(val); CallHelper(slow); + c.End(); // $done + } + + // --------------------------------------------------------------------------------------------------------- + // Inline VFP (single precision) for round-to-nearest and round-to-zero modes, with flush-to-zero / default-NaN handled here. + // Anything involving a NaN (or the one rounding corner we cannot decide) is delegated to the bit-exact soft-float helper, which + // also accumulates the exception flags. The inline path does not update FPSR exception bits. + // --------------------------------------------------------------------------------------------------------- + static constexpr s32 kAbsMask = 0x7fffffff, kSignMask = static_cast(0x80000000u), kInf = 0x7f800000, kMinNormal = 0x00800000; + + void IsNaN32(u32 bits_local) { c.LocalGet(bits_local); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32GtU(); } + void Flush32(u32 src, u32 dst) { // denormal -> signed zero + c.LocalGet(src); c.I32Const(kSignMask); c.I32And(); + c.LocalGet(src); + c.LocalGet(src); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kMinNormal); c.I32LtU(); + c.Select(); c.LocalSet(dst); + } + // bits -= 1 (one ulp toward zero) when `cond` (an i32 on the stack) is non-zero + void StepTowardZeroIf(u32 bits) { + c.If(); + c.LocalGet(bits); c.I32Const(1); c.I32Sub(); c.LocalSet(bits); + c.End(); + } + // if bits is +-inf while all given operands (bit locals) are finite: bits = +-max finite + void FixOverflow(u32 bits, std::initializer_list operands) { + c.LocalGet(bits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32Eq(); + for (u32 o : operands) { c.LocalGet(o); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32LtU(); c.I32And(); } + c.If(); + c.LocalGet(bits); c.I32Const(kSignMask); c.I32And(); c.I32Const(0x7f7fffff); c.I32Or(); c.LocalSet(bits); + c.End(); + } + + // kind: 0 add, 1 sub, 2 mul, 3 div, 4 sqrt + bool FastFp32(IR::Inst* inst, int kind) { + if (!opt.fast_fp || !((opt.fast_fp_mask >> kind) & 1)) return false; + const u32 fpcr = Fpcr(); + const int rm = (fpcr >> 22) & 3; + const bool fz = (fpcr >> 24) & 1; + if (rm != 0 && rm != 3) return false; + const bool unary = kind == 4, rz = rm == 3; + + const u32 ra = Tmp(I32), rb = Tmp(I32); + Ld(inst->GetArg(0)); c.LocalSet(ra); + if (unary) { c.I32Const(0); c.LocalSet(rb); } else { Ld(inst->GetArg(1)); c.LocalSet(rb); } + u32 ia = ra, ib = rb; + if (fz) { ia = Tmp(I32); ib = Tmp(I32); Flush32(ra, ia); if (!unary) Flush32(rb, ib); else { c.I32Const(0); c.LocalSet(ib); } } + + const u32 fa = Tmp(F32), fb = Tmp(F32), rbits = Tmp(I32); + c.LocalGet(ia); c.F32ReinterpretI32(); c.LocalSet(fa); + c.LocalGet(ib); if (kind == 1) { c.I32Const(kSignMask); c.I32Xor(); } c.F32ReinterpretI32(); c.LocalSet(fb); + + if (!rz) { + c.LocalGet(fa); + if (kind == 4) c.F32Sqrt(); else { c.LocalGet(fb); if (kind <= 1) c.F32Add(); else if (kind == 2) c.F32Mul(); else c.F32Div(); } + c.I32ReinterpretF32(); c.LocalSet(rbits); + } else if (kind <= 1) { // add / sub, round toward zero via the exact TwoSum error + const u32 s = Tmp(F32), bb = Tmp(F32), e = Tmp(F32), ebits = Tmp(I32); + c.LocalGet(fa); c.LocalGet(fb); c.F32Add(); c.LocalSet(s); + c.LocalGet(s); c.LocalGet(fa); c.F32Sub(); c.LocalSet(bb); + c.LocalGet(fa); c.LocalGet(s); c.LocalGet(bb); c.F32Sub(); c.F32Sub(); + c.LocalGet(fb); c.LocalGet(bb); c.F32Sub(); c.F32Add(); c.LocalSet(e); + c.LocalGet(s); c.I32ReinterpretF32(); c.LocalSet(rbits); + c.LocalGet(e); c.I32ReinterpretF32(); c.LocalSet(ebits); + // overshoot: finite s, e != 0 and e has the opposite sign of s + c.LocalGet(rbits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32LtU(); + c.LocalGet(ebits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(0); c.I32Ne(); c.I32And(); + c.LocalGet(ebits); c.LocalGet(rbits); c.I32Xor(); c.I32Const(31); c.I32ShrU(); c.I32And(); + StepTowardZeroIf(rbits); + FixOverflow(rbits, {ia, ib}); + } else if (kind == 2) { // mul: the f64 product is exact + const u32 p = Tmp(F64), s = Tmp(F32); + c.LocalGet(fa); c.F64PromoteF32(); c.LocalGet(fb); c.F64PromoteF32(); c.F64Mul(); c.LocalSet(p); + c.LocalGet(p); c.F32DemoteF64(); c.LocalSet(s); + c.LocalGet(s); c.I32ReinterpretF32(); c.LocalSet(rbits); + c.LocalGet(rbits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32LtU(); + c.LocalGet(s); c.F64PromoteF32(); c.F64Abs(); c.LocalGet(p); c.F64Abs(); c.F64Gt(); c.I32And(); + StepTowardZeroIf(rbits); + FixOverflow(rbits, {ia, ib}); + } else if (kind == 3) { // div: remainder a - s*b is exact in f64 + const u32 s = Tmp(F32), rem = Tmp(F64); + c.LocalGet(fa); c.LocalGet(fb); c.F32Div(); c.LocalSet(s); + c.LocalGet(s); c.I32ReinterpretF32(); c.LocalSet(rbits); + c.LocalGet(fa); c.F64PromoteF32(); c.LocalGet(s); c.F64PromoteF32(); c.LocalGet(fb); c.F64PromoteF32(); c.F64Mul(); c.F64Sub(); c.LocalSet(rem); + c.LocalGet(rbits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32LtU(); // s finite + c.LocalGet(rbits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(0); c.I32Ne(); c.I32And(); // s != 0 + c.LocalGet(rem); c.F64Promote0Ne(); c.I32And(); // rem != 0 + c.LocalGet(rem); c.F64LtZero(); c.LocalGet(ib); c.I32Const(31); c.I32ShrU(); c.I32Xor(); // sign(q - s) + c.LocalGet(rbits); c.I32Const(31); c.I32ShrU(); c.I32Ne(); c.I32And(); + StepTowardZeroIf(rbits); + // overflow only when the divisor is a finite non-zero number (x/0 is a true infinity) + c.LocalGet(rbits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32Eq(); + c.LocalGet(ia); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32LtU(); c.I32And(); + c.LocalGet(ib); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32LtU(); c.I32And(); + c.LocalGet(ib); c.I32Const(kAbsMask); c.I32And(); c.I32Const(0); c.I32Ne(); c.I32And(); + c.If(); + c.LocalGet(rbits); c.I32Const(kSignMask); c.I32And(); c.I32Const(0x7f7fffff); c.I32Or(); c.LocalSet(rbits); + c.End(); + } else { // sqrt + const u32 s = Tmp(F32), rem = Tmp(F64); + c.LocalGet(fa); c.F32Sqrt(); c.LocalSet(s); + c.LocalGet(s); c.I32ReinterpretF32(); c.LocalSet(rbits); + c.LocalGet(fa); c.F64PromoteF32(); c.LocalGet(s); c.F64PromoteF32(); c.LocalGet(s); c.F64PromoteF32(); c.F64Mul(); c.F64Sub(); c.LocalSet(rem); + c.LocalGet(rbits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kInf); c.I32LtU(); + c.LocalGet(rem); c.F64LtZero(); c.I32And(); + StepTowardZeroIf(rbits); + } + if (fz) { // flush a denormal result + c.LocalGet(rbits); c.I32Const(kSignMask); c.I32And(); + c.LocalGet(rbits); + c.LocalGet(rbits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kMinNormal); c.I32LtU(); + c.Select(); c.LocalSet(rbits); + } + // Delegate to the soft-float helper if a NaN is involved, or (round-to-nearest + flush) the result is exactly the smallest + // normal number: the architecture flushes on the *unrounded* magnitude, which we cannot see here. + IsNaN32(ra); IsNaN32(rb); c.I32Or(); IsNaN32(rbits); c.I32Or(); + if (fz && !rz) { c.LocalGet(rbits); c.I32Const(kAbsMask); c.I32And(); c.I32Const(kMinNormal); c.I32Eq(); c.I32Or(); } + c.If(I32); + PushCtx(); c.I32Const(static_cast(fpcr)); c.LocalGet(ra); + if (!unary) c.LocalGet(rb); + { static const char* names[] = {"fadd32", "fsub32", "fmul32", "fdiv32", "fsqrt32"}; CallHelper(names[kind]); } + c.Else(); + c.LocalGet(rbits); + c.End(); + Set(inst); + return true; + } + + // VCMP: with no NaN involved the result is a plain ordered comparison of the (flushed) operands. + bool FastCompare32(IR::Inst* inst) { + if (!opt.fast_fp || !((opt.fast_fp_mask >> 5) & 1)) return false; + const u32 fpcr = Fpcr(); + const bool fz = (fpcr >> 24) & 1; + const u32 ra = Tmp(I32), rb = Tmp(I32); + Ld(inst->GetArg(0)); c.LocalSet(ra); + Ld(inst->GetArg(1)); c.LocalSet(rb); + u32 ia = ra, ib = rb; + if (fz) { ia = Tmp(I32); ib = Tmp(I32); Flush32(ra, ia); Flush32(rb, ib); } + const u32 fa = Tmp(F32), fb = Tmp(F32); + c.LocalGet(ia); c.F32ReinterpretI32(); c.LocalSet(fa); + c.LocalGet(ib); c.F32ReinterpretI32(); c.LocalSet(fb); + IsNaN32(ra); IsNaN32(rb); c.I32Or(); + c.If(I32); + PushCtx(); c.I32Const(static_cast(fpcr)); c.LocalGet(ra); c.LocalGet(rb); Ld(inst->GetArg(2)); CallHelper("fcmp32"); + c.Else(); + c.I32Const(0x60000000); // equal + c.I32Const(static_cast(0x80000000u)); c.I32Const(0x20000000); c.LocalGet(fa); c.LocalGet(fb); c.F32Lt(); c.Select(); // less / greater + c.LocalGet(fa); c.LocalGet(fb); c.F32Eq(); + c.Select(); + c.End(); + Set(inst); + return true; + } + + void EmitCoproc(IR::Inst* inst); + + // ---------- module assembly ---------- + static ValType SigChar(char ch) { return ch == 'l' ? I64 : I32; } + + u32 prefix_len = 0, code_off = 0, code_len = 0; // where, inside the assembled module, the shared prefix ends and the function entry (size + body) sits + + Bytes AssembleLegacy() { + // type section: type 0 = block function (i32) -> i32; then one per distinct helper signature + std::vector sigs; // signature strings, index = type index - 1 + auto type_of = [&](const std::string& s) -> u32 { + for (size_t i = 0; i < sigs.size(); i++) if (sigs[i] == s) return static_cast(i + 1); + sigs.push_back(s); + return static_cast(sigs.size()); + }; + std::vector import_types; + for (const auto& name : import_order) { + const char* sig = nullptr; + for (const auto& h : kHelpers) if (name == h.name) sig = h.sig; + if (!sig) { Fail("unknown helper " + name); return {}; } + import_types.push_back(type_of(sig)); + } + Bytes types; + LebU(types, 1 + sigs.size()); + Append(types, Bytes{0x60, 1, I32, 1, I32}); + for (const auto& s : sigs) { + const size_t gt = s.find('>'); + Bytes t{0x60}; + LebU(t, gt); + for (size_t i = 0; i < gt; i++) t.push_back(SigChar(s[i])); + if (s[gt + 1] == 'v') t.push_back(0); + else { t.push_back(1); t.push_back(SigChar(s[gt + 1])); } + Append(types, t); + } + + Bytes imports; + LebU(imports, (uses_table ? 2 : 1) + import_order.size()); + Append(imports, Name("e")); Append(imports, Name("m")); imports.push_back(0x02); imports.push_back(0x00); LebU(imports, 1); + if (uses_table) { Append(imports, Name("e")); Append(imports, Name("t")); imports.push_back(0x01); imports.push_back(0x70); imports.push_back(0x00); LebU(imports, 0); } + for (size_t i = 0; i < import_order.size(); i++) { + Append(imports, Name("e")); Append(imports, Name(import_order[i].c_str())); + imports.push_back(0x00); LebU(imports, import_types[i]); + } + + Bytes funcs{1, 0}; + Bytes exports; + LebU(exports, 1); + Append(exports, Name("b")); exports.push_back(0x00); LebU(exports, import_order.size()); + + // code: local declarations (run-length) + body + end + Bytes body; + std::vector> groups; + for (u8 t : extra_locals) { + if (!groups.empty() && groups.back().second == t) groups.back().first++; + else groups.push_back({1, t}); + } + LebU(body, groups.size()); + for (auto& g : groups) { LebU(body, g.first); body.push_back(g.second); } + Append(body, c.b); + body.push_back(0x0b); + Bytes code; + LebU(code, 1); + LebU(code, body.size()); + Append(code, body); + + Bytes m{0x00, 0x61, 0x73, 0x6d, 1, 0, 0, 0}; + Append(m, Section(1, types)); + Append(m, Section(2, imports)); + Append(m, Section(3, funcs)); + Append(m, Section(7, exports)); + Append(m, Section(10, code)); + return m; + } + + Bytes Assemble() { + // type section: type 0 = block function (i32) -> i32; then type i + 1 = signature of kHelpers[i] + constexpr size_t kN = std::size(kHelpers); + Bytes types; + LebU(types, 1 + kN); + Append(types, Bytes{0x60, 1, I32, 1, I32}); + for (const auto& h : kHelpers) { + const std::string s = h.sig; + const size_t gt = s.find('>'); + Bytes t{0x60}; + LebU(t, gt); + for (size_t i = 0; i < gt; i++) t.push_back(SigChar(s[i])); + if (s[gt + 1] == 'v') t.push_back(0); + else { t.push_back(1); t.push_back(SigChar(s[gt + 1])); } + Append(types, t); + } + + Bytes imports; + LebU(imports, 3); + Append(imports, Name("e")); Append(imports, Name("m")); imports.push_back(0x02); imports.push_back(0x00); LebU(imports, 1); + Append(imports, Name("e")); Append(imports, Name("t")); imports.push_back(0x01); imports.push_back(0x70); imports.push_back(0x00); LebU(imports, 0); // table 0: the block table + Append(imports, Name("e")); Append(imports, Name("h")); imports.push_back(0x01); imports.push_back(0x70); imports.push_back(0x00); LebU(imports, 0); // table 1: the host's function table + + Bytes funcs{1, 0}; + Bytes exports; + LebU(exports, 1); + Append(exports, Name("b")); exports.push_back(0x00); LebU(exports, 0); // no function imports: our function is number 0 + + // code: local declarations (run-length) + body + end + Bytes body; + std::vector> groups; + for (u8 t : extra_locals) { + if (!groups.empty() && groups.back().second == t) groups.back().first++; + else groups.push_back({1, t}); + } + LebU(body, groups.size()); + for (auto& g : groups) { LebU(body, g.first); body.push_back(g.second); } + Append(body, c.b); + body.push_back(0x0b); + Bytes entry; + LebU(entry, body.size()); + Append(entry, body); + Bytes code; + LebU(code, 1); + Append(code, entry); + + Bytes m{0x00, 0x61, 0x73, 0x6d, 1, 0, 0, 0}; + Append(m, Section(1, types)); + Append(m, Section(2, imports)); + prefix_len = static_cast(m.size()); + Append(m, Section(3, funcs)); + Append(m, Section(7, exports)); + Append(m, Section(10, code)); + code_len = static_cast(entry.size()); + code_off = static_cast(m.size() - entry.size()); + return m; + } +}; + +void Emitter::EmitCoproc(IR::Inst* inst) { + const auto info = inst->GetArg(0).GetCoprocInfo(); + const size_t coproc_num = info[0]; + const bool two = info[1] == 1; + const auto& cp = conf.coprocessors[coproc_num]; + if (!cp) { Fail("coprocessor not registered"); return; } + const IR::Opcode op = inst->GetOpcode(); + const auto opc1 = static_cast(info[2]); + const auto CRn = static_cast(info[3]); + const auto CRm = static_cast(info[4]); + const auto opc2 = static_cast(info[5]); + + auto push_callback = [&](const A32::Coprocessor::Callback& cb) { + PushCtx(); + c.I32Const(static_cast(reinterpret_cast(cb.function))); + c.I32Const(static_cast(cb.user_arg ? reinterpret_cast(*cb.user_arg) : 0)); + }; + + switch (op) { + case Opcode::A32CoprocGetOneWord: { + const auto action = cp->CompileGetOneWord(two, opc1, CRn, CRm, opc2); + if (auto* ptr = std::get_if(&action)) { + c.I32Const(static_cast(reinterpret_cast(*ptr))); + c.I32Load(0); + Set(inst); + } else if (auto* cb = std::get_if(&action)) { + push_callback(*cb); + c.I32Const(0); c.I32Const(0); + CallHelper("cpget1"); + Set(inst); + } else { + Fail("coprocessor read not supported for this register"); + } + return; + } + case Opcode::A32CoprocSendOneWord: { + const auto action = cp->CompileSendOneWord(two, opc1, CRn, CRm, opc2); + if (auto* ptr = std::get_if(&action)) { + c.I32Const(static_cast(reinterpret_cast(*ptr))); + Ld(inst->GetArg(1)); + c.I32Store(0); + } else if (auto* cb = std::get_if(&action)) { + push_callback(*cb); + Ld(inst->GetArg(1)); c.I32Const(0); + CallHelper("cpcall"); + } else { + Fail("coprocessor write not supported for this register"); + } + return; + } + default: + Fail("unsupported coprocessor operation " + IR::GetNameOf(op)); + } +} + +} // namespace + +EmitResult EmitA32Block(IR::Block& block, const A32::UserConfig& conf, const EmitOptions& opt) { + return Emitter(block, conf, opt).Run(); +} + +} // namespace Dynarmic::Backend::Wasm diff --git a/src/dynarmic/backend/wasm/emit_a32.h b/src/dynarmic/backend/wasm/emit_a32.h new file mode 100644 index 0000000..f1c38be --- /dev/null +++ b/src/dynarmic/backend/wasm/emit_a32.h @@ -0,0 +1,72 @@ +/* This file is part of the panda3ds-wasm project. + * SPDX-License-Identifier: GPL-3.0-or-later + */ +#pragma once + +#include +#include + +#include + +#include "dynarmic/interface/A32/config.h" +#include "dynarmic/ir/basic_block.h" + +namespace Dynarmic::Backend::Wasm { + +// Host functions that generated blocks may import (module "e"). Signature: params '>' result; +// i = i32, l = i64, v = none. The first parameter of every helper is the Jit::Impl* context. +struct HelperDef { + const char* name; + const char* sig; +}; + +inline constexpr HelperDef kHelpers[] = { + {"r8", "ii>i"}, {"r16", "ii>i"}, {"r32", "ii>i"}, {"r64", "ii>l"}, + {"w8", "iii>v"}, {"w16", "iii>v"}, {"w32", "iii>v"}, {"w64", "iil>v"}, + {"xr8", "ii>i"}, {"xr16", "ii>i"}, {"xr32", "ii>i"}, {"xr64", "ii>l"}, + {"xw8", "iii>i"}, {"xw16", "iii>i"}, {"xw32", "iii>i"}, {"xw64", "iil>i"}, + {"clrex", "i>v"}, + {"svc", "ii>v"}, {"exc", "iii>v"}, {"interp", "iii>v"}, {"isb", "i>v"}, + {"getcpsr", "i>i"}, {"setcpsr", "ii>v"}, {"getfpscr", "i>i"}, {"setfpscr", "ii>v"}, + {"cpget1", "iiiii>i"}, // ctx, callback fn, user_arg, arg0, arg1 -> u32 + {"cpcall", "iiiii>v"}, // ctx, callback fn, user_arg, arg0, arg1 (result ignored) + // VFP scalar arithmetic: (ctx, fpcr, a, b) -> result; FPSR cumulative flags are updated in the state + {"fadd32", "iiii>i"}, {"fsub32", "iiii>i"}, {"fmul32", "iiii>i"}, {"fdiv32", "iiii>i"}, {"fsqrt32", "iii>i"}, + {"fadd64", "iill>l"}, {"fsub64", "iill>l"}, {"fmul64", "iill>l"}, {"fdiv64", "iill>l"}, {"fsqrt64", "iil>l"}, + {"fcmp32", "iiiii>i"}, {"fcmp64", "iilli>i"}, + {"pk", "iii>i"}, // packed halving / saturating ops: (kind, a, b) -> result, see H_pk + {"fs2d", "iiii>l"}, {"fd2s", "iili>i"}, + {"ftoi", "iili>i"}, // (ctx, fpcr, value as i64, mode) -> int bits + {"itof", "iiii>l"}, // (ctx, fpcr, int bits, mode) -> float bits (zero-extended) +}; + +struct EmitResult { + std::vector bytes; + std::string error; // empty on success + u32 prefix_len = 0; ///< bytes[0, prefix_len) = header + type + import sections, identical for every block + u32 code_off = 0, code_len = 0; ///< bytes[code_off, +code_len) = the function entry (size + locals + body), movable into a module with many functions + bool ok() const { return error.empty(); } +}; + +// Block chaining: blocks end by tail-calling the next block through a shared function table, found via a direct-mapped +// cache of {pc, upper location bits, table slot} entries that lives in linear memory. +struct EmitOptions { + u32 link_base = 0; ///< address of the link cache (array of LinkEntry) + u32 link_mask = 0; ///< number of entries - 1 (power of two minus one) + bool chain = false; + bool chain_tail = false; ///< true: wasm tail calls (return_call_indirect); false: nested call_indirect with a depth limit + u32 chain_depth_limit = 160; + u32 fast_fp_mask = 0x3f; ///< which inline VFP ops are enabled (bit per kind: add, sub, mul, div, sqrt, cmp) - for bisecting engine problems + const u32* helper_fn = nullptr; ///< function-table index of each kHelpers entry in the host module (for call_indirect) + bool legacy_layout = false; ///< one minimal module per block (only the imports it uses); the blocks of such modules cannot be packed together + bool fast_fp = false; ///< inline VFP for round-to-nearest / round-to-zero modes (exception flags are not accumulated on this path) +}; +struct LinkEntry { + u32 pc, upper, slot, pad; +}; + +// Translates one IR block to a WebAssembly module exporting `b(state: i32) -> i32` +// (0 = continue with PC from state, 1 = stop/return to the caller). +EmitResult EmitA32Block(IR::Block& block, const A32::UserConfig& conf, const EmitOptions& opt = {}); + +} // namespace Dynarmic::Backend::Wasm diff --git a/src/dynarmic/backend/wasm/exclusive_monitor.cpp b/src/dynarmic/backend/wasm/exclusive_monitor.cpp new file mode 100644 index 0000000..c1651b4 --- /dev/null +++ b/src/dynarmic/backend/wasm/exclusive_monitor.cpp @@ -0,0 +1,66 @@ +/* This file is part of the dynarmic project. + * Copyright (c) 2018 MerryMage + * SPDX-License-Identifier: 0BSD + */ + +#include "dynarmic/interface/exclusive_monitor.h" + +#include + +#include + +namespace Dynarmic { + +ExclusiveMonitor::ExclusiveMonitor(size_t processor_count) + : exclusive_addresses(processor_count, INVALID_EXCLUSIVE_ADDRESS), exclusive_values(processor_count) {} + +size_t ExclusiveMonitor::GetProcessorCount() const { + return exclusive_addresses.size(); +} + +void ExclusiveMonitor::Lock() { + lock.Lock(); +} + +void ExclusiveMonitor::Unlock() { + lock.Unlock(); +} + +bool ExclusiveMonitor::CheckAndClear(size_t processor_id, VAddr address) { + const VAddr masked_address = address & RESERVATION_GRANULE_MASK; + + Lock(); + if (exclusive_addresses[processor_id] != masked_address) { + Unlock(); + return false; + } + + for (VAddr& other_address : exclusive_addresses) { + if (other_address == masked_address) { + other_address = INVALID_EXCLUSIVE_ADDRESS; + } + } + return true; +} + +void ExclusiveMonitor::Clear() { + Lock(); + std::fill(exclusive_addresses.begin(), exclusive_addresses.end(), INVALID_EXCLUSIVE_ADDRESS); + Unlock(); +} + +void ExclusiveMonitor::ClearProcessor(size_t processor_id) { + Lock(); + exclusive_addresses[processor_id] = INVALID_EXCLUSIVE_ADDRESS; + Unlock(); +} + +} // namespace Dynarmic + +// SpinLock for WebAssembly: Dynarmic's guest runs on one thread here, so a plain atomic flag is enough. +#include "dynarmic/common/spin_lock.h" + +namespace Dynarmic { +void SpinLock::Lock() { while (__atomic_exchange_n(&storage, 1, __ATOMIC_ACQUIRE) != 0) {} } +void SpinLock::Unlock() { __atomic_store_n(&storage, 0, __ATOMIC_RELEASE); } +} // namespace Dynarmic diff --git a/src/dynarmic/backend/wasm/fp_soft.cpp b/src/dynarmic/backend/wasm/fp_soft.cpp new file mode 100644 index 0000000..a04dbbd --- /dev/null +++ b/src/dynarmic/backend/wasm/fp_soft.cpp @@ -0,0 +1,237 @@ +/* This file is part of the panda3ds-wasm project. + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +#include "dynarmic/backend/wasm/fp_soft.h" + +#include + +#include "dynarmic/common/fp/fused.h" +#include "dynarmic/common/fp/info.h" +#include "dynarmic/common/fp/op/FPConvert.h" +#include "dynarmic/common/fp/op/FPToFixed.h" +#include "dynarmic/common/fp/process_exception.h" +#include "dynarmic/common/fp/process_nan.h" +#include "dynarmic/common/fp/unpacked.h" + +namespace Dynarmic::Backend::Wasm::FPSoft { + +using namespace Dynarmic::FP; + +namespace { + +constexpr FPUnpacked kOne = {false, 0, u64(1) << normalized_point_position}; + +template +FPT AddSub(FPT op1, FPT op2, bool negate2, FPCR fpcr, FPSR& fpsr) { + const RoundingMode rounding = fpcr.RMode(); + auto [type1, sign1, value1] = FPUnpack(op1, fpcr, fpsr); + auto [type2, sign2, value2] = FPUnpack(op2, fpcr, fpsr); + + // NaN operands propagate unchanged (a negated NaN keeps its own sign), so look at them before negating. + if (const auto nan = FPProcessNaNs(type1, type2, op1, op2, fpcr, fpsr)) return *nan; + + if (negate2) { + sign2 = !sign2; + value2.sign = !value2.sign; + } + const bool inf1 = type1 == FPType::Infinity, inf2 = type2 == FPType::Infinity; + const bool zero1 = type1 == FPType::Zero, zero2 = type2 == FPType::Zero; + + if (inf1 && inf2 && sign1 == !sign2) { + FPProcessException(FPExc::InvalidOp, fpcr, fpsr); + return FPInfo::DefaultNaN(); + } + if ((inf1 && !sign1) || (inf2 && !sign2)) return FPInfo::Infinity(false); + if ((inf1 && sign1) || (inf2 && sign2)) return FPInfo::Infinity(true); + if (zero1 && zero2 && sign1 == sign2) return FPInfo::Zero(sign1); + + const FPUnpacked result = FusedMulAdd(value1, value2, kOne); // value1 + value2 * 1, exact + if (result.mantissa == 0) return FPInfo::Zero(rounding == RoundingMode::TowardsMinusInfinity); + return FPRound(result, fpcr, fpsr); +} + +// Unsigned 128/64 -> quotient and "remainder != 0" without 128-bit division intrinsics. +inline bool DivideNumerator(u64 num_hi, u64 num_lo, u64 den, u64& quotient) { + // Restoring division of the 128-bit value (num_hi:num_lo) by den; quotient must fit in 64 bits (num_hi < den). + u64 rem = num_hi; + u64 q = 0; + for (int i = 63; i >= 0; i--) { + const bool carry = rem >> 63; + rem = (rem << 1) | ((num_lo >> i) & 1); + q <<= 1; + if (carry || rem >= den) { + rem -= den; + q |= 1; + } + } + quotient = q; + return rem != 0; +} + +inline u64 IntSqrt128(u64 hi, u64 lo, bool& inexact) { + // floor(sqrt(hi:lo)) by the classic restoring method over bit pairs; hi:lo < 2^126 so the result fits in 63 bits. + unsigned __int128 x = (static_cast(hi) << 64) | lo, res = 0, bit = static_cast(1) << 126; + while (bit > x) bit >>= 2; + while (bit != 0) { + if (x >= res + bit) { + x -= res + bit; + res = (res >> 1) + bit; + } else { + res >>= 1; + } + bit >>= 2; + } + inexact = x != 0; + return static_cast(res); +} + +} // namespace + +template +FPT Add(FPT a, FPT b, FPCR fpcr, FPSR& fpsr) { return AddSub(a, b, false, fpcr, fpsr); } +template +FPT Sub(FPT a, FPT b, FPCR fpcr, FPSR& fpsr) { return AddSub(a, b, true, fpcr, fpsr); } + +template +FPT Mul(FPT op1, FPT op2, FPCR fpcr, FPSR& fpsr) { + const auto [type1, sign1, value1] = FPUnpack(op1, fpcr, fpsr); + const auto [type2, sign2, value2] = FPUnpack(op2, fpcr, fpsr); + if (const auto nan = FPProcessNaNs(type1, type2, op1, op2, fpcr, fpsr)) return *nan; + + const bool inf1 = type1 == FPType::Infinity, inf2 = type2 == FPType::Infinity; + const bool zero1 = type1 == FPType::Zero, zero2 = type2 == FPType::Zero; + if ((inf1 && zero2) || (zero1 && inf2)) { + FPProcessException(FPExc::InvalidOp, fpcr, fpsr); + return FPInfo::DefaultNaN(); + } + if (inf1 || inf2) return FPInfo::Infinity(sign1 != sign2); + if (zero1 || zero2) return FPInfo::Zero(sign1 != sign2); + + const FPUnpacked zero = {false, 0, 0}; + return FPRound(FusedMulAdd(zero, value1, value2), fpcr, fpsr); +} + +template +FPT Div(FPT op1, FPT op2, FPCR fpcr, FPSR& fpsr) { + const auto [type1, sign1, value1] = FPUnpack(op1, fpcr, fpsr); + const auto [type2, sign2, value2] = FPUnpack(op2, fpcr, fpsr); + if (const auto nan = FPProcessNaNs(type1, type2, op1, op2, fpcr, fpsr)) return *nan; + + const bool inf1 = type1 == FPType::Infinity, inf2 = type2 == FPType::Infinity; + const bool zero1 = type1 == FPType::Zero, zero2 = type2 == FPType::Zero; + if ((inf1 && inf2) || (zero1 && zero2)) { + FPProcessException(FPExc::InvalidOp, fpcr, fpsr); + return FPInfo::DefaultNaN(); + } + if (inf1 || zero2) { + if (!inf1) FPProcessException(FPExc::DivideByZero, fpcr, fpsr); + return FPInfo::Infinity(sign1 != sign2); + } + if (zero1 || inf2) return FPInfo::Zero(sign1 != sign2); + + // value1 / value2 = (m1 / m2) * 2^(e1 - e2) with m in [1,2) represented at point position 62. + // quotient = floor(m1 * 2^62 / m2) as a 64-bit value in (2^61, 2^63); a non-zero remainder becomes a sticky low bit. + u64 q = 0; + const u64 num_hi = value1.mantissa >> 2; // (m1 << 62) as a 128-bit number: high part + const u64 num_lo = value1.mantissa << 62; // low part + const bool inexact = DivideNumerator(num_hi, num_lo, value2.mantissa, q); + const FPUnpacked result = ToNormalized(sign1 != sign2, value1.exponent - value2.exponent, q | static_cast(inexact)); + // ToNormalized treats `q` as an integer scaled by 2^exponent; q already carries 2^-62 scaling, so compensate. + FPUnpacked fixed = result; + fixed.exponent -= static_cast(normalized_point_position); + return FPRound(fixed, fpcr, fpsr); +} + +template +FPT Sqrt(FPT op, FPCR fpcr, FPSR& fpsr) { + const auto [type, sign, value] = FPUnpack(op, fpcr, fpsr); + if (type == FPType::SNaN || type == FPType::QNaN) return FPProcessNaN(type, op, fpcr, fpsr); + if (type == FPType::Zero) return FPInfo::Zero(sign); + if (type == FPType::Infinity && !sign) return FPInfo::Infinity(false); + if (sign) { + FPProcessException(FPExc::InvalidOp, fpcr, fpsr); + return FPInfo::DefaultNaN(); + } + // value = (m / 2^62) * 2^e, m in [2^62, 2^63). Make the exponent even, then sqrt(m' * 2^62) / 2^62 * 2^(e'/2). + int e = value.exponent; + u64 m = value.mantissa; + unsigned __int128 n; + if (e & 1) { // odd exponent (works for negative e: e & 1 is 1 for odd in two's complement) + n = static_cast(m) << 63; // m' = m * 2 + e -= 1; + } else { + n = static_cast(m) << 62; + } + bool inexact = false; + const u64 r = IntSqrt128(static_cast(n >> 64), static_cast(n), inexact); + FPUnpacked result = ToNormalized(false, e / 2, r | static_cast(inexact)); + result.exponent -= static_cast(normalized_point_position); + return FPRound(result, fpcr, fpsr); +} + +template +u32 Compare(FPT a, FPT b, bool exc_on_qnan, FPCR fpcr, FPSR& fpsr) { + const auto [type1, sign1, value1] = FPUnpack(a, fpcr, fpsr); + const auto [type2, sign2, value2] = FPUnpack(b, fpcr, fpsr); + const bool nan1 = type1 == FPType::QNaN || type1 == FPType::SNaN; + const bool nan2 = type2 == FPType::QNaN || type2 == FPType::SNaN; + if (nan1 || nan2) { + if (type1 == FPType::SNaN || type2 == FPType::SNaN || exc_on_qnan) FPProcessException(FPExc::InvalidOp, fpcr, fpsr); + return 0x30000000; // unordered: C and V + } + // Order the (flushed) values. Zero compares equal regardless of sign. + auto cmp = [](FPType t1, bool s1, const FPUnpacked& v1, FPType t2, bool s2, const FPUnpacked& v2) -> int { + auto key_sign = [](FPType t, bool s) { return t == FPType::Zero ? 0 : (s ? -1 : 1); }; + const int k1 = key_sign(t1, s1), k2 = key_sign(t2, s2); + if (k1 != k2) return k1 < k2 ? -1 : 1; + if (k1 == 0) return 0; + // same sign, both non-zero: compare magnitudes + int mag; + if (t1 == FPType::Infinity || t2 == FPType::Infinity) mag = (t1 == FPType::Infinity) - (t2 == FPType::Infinity); + else if (v1.exponent != v2.exponent) mag = v1.exponent < v2.exponent ? -1 : 1; + else mag = v1.mantissa == v2.mantissa ? 0 : (v1.mantissa < v2.mantissa ? -1 : 1); + return k1 > 0 ? mag : -mag; + }; + const int c = cmp(type1, sign1, value1, type2, sign2, value2); + if (c == 0) return 0x60000000; // equal: Z, C + if (c < 0) return 0x80000000; // less: N + return 0x20000000; // greater: C +} + +u64 SingleToDouble(u32 a, FPCR fpcr, RoundingMode rm, FPSR& fpsr) { return FPConvert(a, fpcr, rm, fpsr); } +u32 DoubleToSingle(u64 a, FPCR fpcr, RoundingMode rm, FPSR& fpsr) { return FPConvert(a, fpcr, rm, fpsr); } + +template +u32 ToFixed(FPT a, size_t ibits, size_t fbits, bool is_unsigned, FPCR fpcr, RoundingMode rm, FPSR& fpsr) { + return static_cast(FPToFixed(ibits, a, fbits, is_unsigned, fpcr, rm, fpsr)); +} + +template +FPT FromFixed(u32 value, size_t ibits, size_t fbits, bool is_unsigned, FPCR fpcr, RoundingMode rm, FPSR& fpsr) { + bool sign = false; + u64 mag = value; + if (ibits < 32) mag &= (u64(1) << ibits) - 1; + if (!is_unsigned) { + const u64 sign_bit = u64(1) << (ibits - 1); + if (mag & sign_bit) { + sign = true; + mag = ((u64(1) << ibits) - mag) & ((u64(1) << ibits) - 1); + if (mag == 0) mag = u64(1) << ibits; // most negative value + } + } + if (mag == 0) return FPInfo::Zero(false); + return FPRound(ToNormalized(sign, -static_cast(fbits), mag), fpcr, rm, fpsr); +} + +#define INST(T) \ + template T Add(T, T, FPCR, FPSR&); template T Sub(T, T, FPCR, FPSR&); \ + template T Mul(T, T, FPCR, FPSR&); template T Div(T, T, FPCR, FPSR&); \ + template T Sqrt(T, FPCR, FPSR&); template u32 Compare(T, T, bool, FPCR, FPSR&); \ + template u32 ToFixed(T, size_t, size_t, bool, FPCR, RoundingMode, FPSR&); \ + template T FromFixed(u32, size_t, size_t, bool, FPCR, RoundingMode, FPSR&); +INST(u32) +INST(u64) +#undef INST + +} // namespace Dynarmic::Backend::Wasm::FPSoft diff --git a/src/dynarmic/backend/wasm/fp_soft.h b/src/dynarmic/backend/wasm/fp_soft.h new file mode 100644 index 0000000..20f847b --- /dev/null +++ b/src/dynarmic/backend/wasm/fp_soft.h @@ -0,0 +1,36 @@ +/* This file is part of the panda3ds-wasm project. + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Bit-exact ARM VFP scalar arithmetic on top of Dynarmic's portable soft-float building blocks (FPUnpack/FPRound/FusedMulAdd). + * Handles every FPCR mode (rounding, flush-to-zero, default-NaN) and accumulates exception flags into the FPSR. + */ +#pragma once + +#include + +#include "dynarmic/common/fp/fpcr.h" +#include "dynarmic/common/fp/fpsr.h" +#include "dynarmic/common/fp/rounding_mode.h" + +namespace Dynarmic::Backend::Wasm::FPSoft { + +template FPT Add(FPT a, FPT b, FP::FPCR fpcr, FP::FPSR& fpsr); +template FPT Sub(FPT a, FPT b, FP::FPCR fpcr, FP::FPSR& fpsr); +template FPT Mul(FPT a, FPT b, FP::FPCR fpcr, FP::FPSR& fpsr); +template FPT Div(FPT a, FPT b, FP::FPCR fpcr, FP::FPSR& fpsr); +template FPT Sqrt(FPT a, FP::FPCR fpcr, FP::FPSR& fpsr); + +/// VCMP/VCMPE: returns NZCV in bits 31..28. +template u32 Compare(FPT a, FPT b, bool exc_on_qnan, FP::FPCR fpcr, FP::FPSR& fpsr); + +/// float <-> float +u64 SingleToDouble(u32 a, FP::FPCR fpcr, FP::RoundingMode rm, FP::FPSR& fpsr); +u32 DoubleToSingle(u64 a, FP::FPCR fpcr, FP::RoundingMode rm, FP::FPSR& fpsr); + +/// float -> fixed/integer (ibits <= 32), result left in the low `ibits` bits (two's complement for signed). +template u32 ToFixed(FPT a, size_t ibits, size_t fbits, bool is_unsigned, FP::FPCR fpcr, FP::RoundingMode rm, FP::FPSR& fpsr); + +/// fixed/integer (ibits <= 32) -> float +template FPT FromFixed(u32 value, size_t ibits, size_t fbits, bool is_unsigned, FP::FPCR fpcr, FP::RoundingMode rm, FP::FPSR& fpsr); + +} // namespace Dynarmic::Backend::Wasm::FPSoft diff --git a/src/dynarmic/backend/wasm/wasm_builder.h b/src/dynarmic/backend/wasm/wasm_builder.h new file mode 100644 index 0000000..f0cb5b3 --- /dev/null +++ b/src/dynarmic/backend/wasm/wasm_builder.h @@ -0,0 +1,128 @@ +/* This file is part of the panda3ds-wasm project. + * SPDX-License-Identifier: GPL-3.0-or-later + * Minimal WebAssembly binary writer (MVP + sign-extension ops). + */ +#pragma once + +#include +#include + +#include + +namespace Dynarmic::Backend::Wasm { + +using Bytes = std::vector; + +inline void LebU(Bytes& b, u64 v) { + do { + u8 byte = v & 0x7f; + v >>= 7; + if (v) byte |= 0x80; + b.push_back(byte); + } while (v); +} + +inline void LebS(Bytes& b, s64 v) { + for (;;) { + const u8 byte = v & 0x7f; + v >>= 7; + if ((v == 0 && !(byte & 0x40)) || (v == -1 && (byte & 0x40))) { + b.push_back(byte); + return; + } + b.push_back(byte | 0x80); + } +} + +inline void Append(Bytes& b, const Bytes& o) { b.insert(b.end(), o.begin(), o.end()); } + +inline Bytes Section(u8 id, const Bytes& payload) { + Bytes b{id}; + LebU(b, payload.size()); + Append(b, payload); + return b; +} + +inline Bytes Name(const char* s) { + Bytes b; + size_t n = 0; + while (s[n]) n++; + LebU(b, n); + for (size_t i = 0; i < n; i++) b.push_back(static_cast(s[i])); + return b; +} + +enum ValType : u8 { I32 = 0x7f, I64 = 0x7e, F32 = 0x7d, F64 = 0x7c }; + +// Function body writer. Numeric opcode names follow the spec text format. +struct Code { + Bytes b; + + void Op(u8 o) { b.push_back(o); } + void I32Const(s32 v) { Op(0x41); LebS(b, v); } + void I64Const(s64 v) { Op(0x42); LebS(b, v); } + void LocalGet(u32 i) { Op(0x20); LebU(b, i); } + void LocalSet(u32 i) { Op(0x21); LebU(b, i); } + void LocalTee(u32 i) { Op(0x22); LebU(b, i); } + void Call(u32 f) { Op(0x10); LebU(b, f); } + void CallIndirect(u32 type, u32 table) { Op(0x11); LebU(b, type); LebU(b, table); } + void ReturnCallIndirect(u32 type, u32 table) { Op(0x13); LebU(b, type); LebU(b, table); } + + void I32Load(u32 off) { Op(0x28); Op(2); LebU(b, off); } + void I64Load(u32 off) { Op(0x29); Op(3); LebU(b, off); } + void I32Load8U(u32 off) { Op(0x2d); Op(0); LebU(b, off); } + void I32Load16U(u32 off) { Op(0x2f); Op(1); LebU(b, off); } + void I32Store8(u32 off) { Op(0x3a); Op(0); LebU(b, off); } + void I32Store16(u32 off) { Op(0x3b); Op(1); LebU(b, off); } + void I32Store(u32 off) { Op(0x36); Op(2); LebU(b, off); } + void I64Store(u32 off) { Op(0x37); Op(3); LebU(b, off); } + + void Block(u8 blocktype = 0x40) { Op(0x02); Op(blocktype); } + void Br(u32 depth) { Op(0x0c); LebU(b, depth); } + void BrIf(u32 depth) { Op(0x0d); LebU(b, depth); } + void If(u8 blocktype = 0x40) { Op(0x04); Op(blocktype); } + void Else() { Op(0x05); } + void End() { Op(0x0b); } + void Return() { Op(0x0f); } + void Unreachable() { Op(0x00); } + void Drop() { Op(0x1a); } + void Select() { Op(0x1b); } + + // i32 + void I32Eqz() { Op(0x45); } void I32Eq() { Op(0x46); } void I32Ne() { Op(0x47); } + void I32LtS() { Op(0x48); } void I32LtU() { Op(0x49); } void I32GtS() { Op(0x4a); } void I32GtU() { Op(0x4b); } + void I32LeS() { Op(0x4c); } void I32LeU() { Op(0x4d); } void I32GeS() { Op(0x4e); } void I32GeU() { Op(0x4f); } + void I32Clz() { Op(0x67); } + void I32Add() { Op(0x6a); } void I32Sub() { Op(0x6b); } void I32Mul() { Op(0x6c); } + void I32DivS() { Op(0x6d); } void I32DivU() { Op(0x6e); } + void I32And() { Op(0x71); } void I32Or() { Op(0x72); } void I32Xor() { Op(0x73); } + void I32Shl() { Op(0x74); } void I32ShrS() { Op(0x75); } void I32ShrU() { Op(0x76); } + void I32Rotr() { Op(0x78); } + // i64 + void I64GtS() { Op(0x55); } + void I64Eqz() { Op(0x50); } void I64Eq() { Op(0x51); } void I64Ne() { Op(0x52); } + void I64Add() { Op(0x7c); } void I64Sub() { Op(0x7d); } void I64Mul() { Op(0x7e); } + void I64And() { Op(0x83); } void I64Or() { Op(0x84); } void I64Xor() { Op(0x85); } + void I64Shl() { Op(0x86); } void I64ShrS() { Op(0x87); } void I64ShrU() { Op(0x88); } + // f32 / f64 + void F32Eq() { Op(0x5b); } void F32Ne() { Op(0x5c); } void F32Lt() { Op(0x5d); } void F32Gt() { Op(0x5e); } + void F64Eq() { Op(0x61); } void F64Ne() { Op(0x62); } void F64Lt() { Op(0x63); } void F64Gt() { Op(0x64); } + void F32Abs() { Op(0x8b); } void F32Sqrt() { Op(0x91); } void F32Add() { Op(0x92); } void F32Sub() { Op(0x93); } void F32Mul() { Op(0x94); } void F32Div() { Op(0x95); } + void F64Abs() { Op(0x99); } void F64Sqrt() { Op(0x9f); } void F64Add() { Op(0xa0); } void F64Sub() { Op(0xa1); } void F64Mul() { Op(0xa2); } void F64Div() { Op(0xa3); } + void F64Promote0Ne() { Op(0x44); for (int i = 0; i < 8; i++) Op(0); Op(0x62); } // f64 != 0.0 + void F64LtZero() { Op(0x44); for (int i = 0; i < 8; i++) Op(0); Op(0x63); } // f64 < 0.0 + void F32DemoteF64() { Op(0xb6); } void F64PromoteF32() { Op(0xbb); } + void F32ConvertI32S() { Op(0xb2); } void F32ConvertI32U() { Op(0xb3); } void F64ConvertI32S() { Op(0xb7); } void F64ConvertI32U() { Op(0xb8); } + void I32ReinterpretF32() { Op(0xbc); } void F32ReinterpretI32() { Op(0xbe); } + void I64ReinterpretF64() { Op(0xbd); } void F64ReinterpretI64() { Op(0xbf); } + void I32TruncSatF32S() { Op(0xfc); Op(0x00); } void I32TruncSatF32U() { Op(0xfc); Op(0x01); } + void I32TruncSatF64S() { Op(0xfc); Op(0x02); } void I32TruncSatF64U() { Op(0xfc); Op(0x03); } + // conversions + void I32WrapI64() { Op(0xa7); } + void I64ExtendI32S() { Op(0xac); } + void I64ExtendI32U() { Op(0xad); } + void I32Extend8S() { Op(0xc0); } + void I32Extend16S() { Op(0xc1); } +}; + +} // namespace Dynarmic::Backend::Wasm diff --git a/src/dynarmic/common/fp/op/FPRecipExponent.cpp b/src/dynarmic/common/fp/op/FPRecipExponent.cpp index 171b94e..9ed8637 100644 --- a/src/dynarmic/common/fp/op/FPRecipExponent.cpp +++ b/src/dynarmic/common/fp/op/FPRecipExponent.cpp @@ -17,7 +17,7 @@ namespace Dynarmic::FP { namespace { template -FPT DetermineExponentValue(size_t value) { +FPT DetermineExponentValue(u64 value) { if constexpr (sizeof(FPT) == sizeof(u32)) { return static_cast(mcl::bit::get_bits<23, 30>(value)); } else if constexpr (sizeof(FPT) == sizeof(u64)) { diff --git a/src/dynarmic/common/fp/op/FPToFixed.cpp b/src/dynarmic/common/fp/op/FPToFixed.cpp index b1a57d3..19d5c13 100644 --- a/src/dynarmic/common/fp/op/FPToFixed.cpp +++ b/src/dynarmic/common/fp/op/FPToFixed.cpp @@ -38,11 +38,6 @@ u64 FPToFixed(size_t ibits, FPT op, size_t fbits, bool unsigned_, FPCR fpcr, Rou return 0; } - if (sign && unsigned_) { - FPProcessException(FPExc::InvalidOp, fpcr, fpsr); - return 0; - } - // value *= 2.0^fbits and reshift the decimal point back to bit zero. int exponent = value.exponent + static_cast(fbits) - normalized_point_position; @@ -75,6 +70,20 @@ u64 FPToFixed(size_t ibits, FPT op, size_t fbits, bool unsigned_, FPCR fpcr, Rou int_result++; } + // Unsigned conversion of a negative value: a result that rounds to zero is only inexact; anything else is invalid and saturates to zero. + // (Verified against Apple Silicon FCVTZU/FCVTNU: the check must come after rounding.) + if (sign && unsigned_) { + // |value| >= 1 (mantissa has bit 62 set, so exponent >= -62) can never round to zero. + if (exponent >= -static_cast(normalized_point_position) || int_result != 0) { + FPProcessException(FPExc::InvalidOp, fpcr, fpsr); + return 0; + } + if (error != ResidualError::Zero) { + FPProcessException(FPExc::Inexact, fpcr, fpsr); + } + return 0; + } + // Detect Overflow const int min_exponent_for_overflow = static_cast(ibits) - static_cast(mcl::bit::highest_set_bit(value.mantissa + (round_up ? Safe::LogicalShiftRight(1, exponent) : 0))) - (unsigned_ ? 0 : 1); if (exponent >= min_exponent_for_overflow) { diff --git a/src/dynarmic/common/llvm_disassemble.cpp b/src/dynarmic/common/llvm_disassemble.cpp index 636ee3a..2861b7d 100644 --- a/src/dynarmic/common/llvm_disassemble.cpp +++ b/src/dynarmic/common/llvm_disassemble.cpp @@ -50,7 +50,7 @@ std::string DisassembleX64(const void* begin, const void* end) { LLVMDisasmDispose(llvm_ctx); #else result += fmt::format("(recompile with DYNARMIC_USE_LLVM=ON to disassemble the generated x86_64 code)\n"); - result += fmt::format("start: {:016x}, end: {:016x}\n", mcl::bit_cast(begin), mcl::bit_cast(end)); + result += fmt::format("start: {:016x}, end: {:016x}\n", static_cast(reinterpret_cast(begin)), static_cast(reinterpret_cast(end))); #endif return result; diff --git a/src/dynarmic/interface/A32/config.h b/src/dynarmic/interface/A32/config.h index 360df06..b0b1e59 100644 --- a/src/dynarmic/interface/A32/config.h +++ b/src/dynarmic/interface/A32/config.h @@ -156,6 +156,9 @@ struct UserConfig { static constexpr std::size_t PAGE_BITS = 12; static constexpr std::size_t NUM_PAGE_TABLE_ENTRIES = 1 << (32 - PAGE_BITS); std::array* page_table = nullptr; + /// WebAssembly backend (wasm32 only): separate read/write page tables (one 32-bit host pointer per 4 KiB page, 0 = use the callbacks). + const std::uint32_t* wasm_read_table = nullptr; + const std::uint32_t* wasm_write_table = nullptr; /// Determines if the pointer in the page_table shall be offseted locally or globally. /// 'false' will access page_table[addr >> bits][addr & mask] /// 'true' will access page_table[addr >> bits][addr] diff --git a/src/dynarmic/ir/ir_emitter.cpp b/src/dynarmic/ir/ir_emitter.cpp index fc4f69b..c1de9a5 100644 --- a/src/dynarmic/ir/ir_emitter.cpp +++ b/src/dynarmic/ir/ir_emitter.cpp @@ -2868,19 +2868,19 @@ void IREmitter::Breakpoint() { } void IREmitter::CallHostFunction(void (*fn)(void)) { - Inst(Opcode::CallHostFunction, Imm64(mcl::bit_cast(fn)), Value{}, Value{}, Value{}); + Inst(Opcode::CallHostFunction, Imm64(static_cast(reinterpret_cast(fn))), Value{}, Value{}, Value{}); } void IREmitter::CallHostFunction(void (*fn)(u64), const U64& arg1) { - Inst(Opcode::CallHostFunction, Imm64(mcl::bit_cast(fn)), arg1, Value{}, Value{}); + Inst(Opcode::CallHostFunction, Imm64(static_cast(reinterpret_cast(fn))), arg1, Value{}, Value{}); } void IREmitter::CallHostFunction(void (*fn)(u64, u64), const U64& arg1, const U64& arg2) { - Inst(Opcode::CallHostFunction, Imm64(mcl::bit_cast(fn)), arg1, arg2, Value{}); + Inst(Opcode::CallHostFunction, Imm64(static_cast(reinterpret_cast(fn))), arg1, arg2, Value{}); } void IREmitter::CallHostFunction(void (*fn)(u64, u64, u64), const U64& arg1, const U64& arg2, const U64& arg3) { - Inst(Opcode::CallHostFunction, Imm64(mcl::bit_cast(fn)), arg1, arg2, arg3); + Inst(Opcode::CallHostFunction, Imm64(static_cast(reinterpret_cast(fn))), arg1, arg2, arg3); } void IREmitter::SetTerm(const Terminal& terminal) {