PlayAgain 0103: MEM2 (the Wii's second RAM) inline, in compiled blocks and in the C++ paths of the blocks too short to compile (design J1 item 1). SPDX-License-Identifier: GPL-2.0-or-later Tree: vendor/dolphin (Dolphin e22551ea + the fork's 150 locked patches + PlayAgain 0101). What: the fork reaches MEM1 inline through the 0x8/0xC views (the standard BATs assumed, no data-cache emulation, no run-time BAT check) and sends every MEM2 access (0x9/0xD) to the MMU: 99.6% of Wii Sports bowling's slow integer loads (J0). Now a failed MEM1 test falls through to the same test on MEM2: ((a ^ 0x90000000) & 0xbfffffff) <= exram_size_real - size, then the MEM1 arm's own instructions on MEM2's base, in every inline memory emitter of the block and fragment emitters (integer loads and stores, lfs/lfd/stfs/stfd/stfiwx, lmw/stmw, psq_l/psq_st float, dcbz) and in TryFastRamWord/ByteHalf/ Multiword/Float and the psq C++ path (PaFastRamView). Same guard as MEM1. A store takes the cached 0x9 view only: a sub-word or unaligned store through the uncached 0xD view spreads over the 8-byte line and raises a PI interrupt in MMU::WriteToHardware (Dolphin issue 12565), which a plain copy would not do. Blocks embed MEM2's base as a constant (as MEM1's); region fragments load it from a .bss cell whose address the layout hash covers. GameCube and ?mem2inline=0: no MEM2 arm, the same bytes as before. The oracle gets a Wii's memory and MEM2 states (DolphinWeb_PaOracleMem2ForTest: scratch in MEM1, MEM2's middle, its last 256 bytes, its first 256; DolphinWeb_PaOracleWindowForTest for the region oracle). Switch: ?mem2inline=0 (bit 1 of the cached-interpreter mask, DISABLE polarity, set by 0103-fork). --- a/Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreter.cpp +++ b/Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreter.cpp @@ -401,7 +401,11 @@ // (and wherever relaxed SIMD is missing or unfused): the plain-f64 path. std::atomic s_fused_madd{false}; -// Bits 1 and 2 are unused (patch 0196 removed the helpers they gated). +// Bit 2 is unused (patch 0196 removed the helpers bits 1 and 2 gated). +// PlayAgain 0103: DISABLE polarity (default-on; ?mem2inline=0). Set: MEM2 (the Wii's 64 MiB) is +// reached only through the MMU helpers again, as before 0103 (compiled code and the C++ paths +// of the blocks too short to compile alike). +constexpr u32 DOLPHIN_WEB_DISABLE_PA_MEM2_INLINE = 1u << 1; // PlayAgain 0101: ENABLE polarity (default-off; ?dcbxskip=legacy). Set: with the wasm JIT off, // every dcbx-loop invalidation is skipped again, as before 0101 (the fork's behaviour at 50f0c362). constexpr u32 DOLPHIN_WEB_ENABLE_LEGACY_DCBX_SKIP = 1u << 0; @@ -1286,6 +1290,47 @@ static u32 Mem1ViewOffset(u32 address) { return (address ^ 0x80000000u) & 0xbfffffffu; +} + +// PlayAgain 0103: the MEM2 offset of an address in the 0x90000000 (cached) or 0xD0000000 +// (uncached) view, the views the standard Wii BATs map onto MEM2 (physical 0x10000000); any +// other address yields a value >= 0x10000000, so the caller's "offset < MEM2 size" test sends +// it to the MMU, as Mem1ViewOffset does for MEM1. +static u32 Mem2ViewOffset(u32 address) +{ + return (address ^ 0x90000000u) & 0xbfffffffu; +} + +// PlayAgain 0103: where the C++ fast paths of uncompiled blocks find [address, address + bytes): +// MEM1 (0x8/0xC views), or on a Wii with MEM2 inline on, MEM2 (0x9/0xD views for a load, the +// cached 0x9 view only for a store); a null base sends the access to the MMU. MEM2 keeps MEM1's +// guard: the standard BATs are assumed (only those views are RAM; any other address, real +// mode's physical ones included, takes the MMU) and the data cache is not emulated (MMU.cpp +// copies plainly then). Stores through the uncached 0xD view stay with the MMU: a sub-word or +// unaligned cache-inhibited store spreads its data over the 8-byte line and raises a PI +// interrupt there (MMU::WriteToHardware, Dolphin issue 12565), which a plain copy would not. +struct PaRamView +{ + u8* base; + u32 offset; +}; +static PaRamView PaFastRamView(Core::System& system, u32 address, u32 bytes, bool store) +{ + auto& memory = system.GetMemory(); + u8* const ram = memory.GetRAM(); + const u32 ram_size = memory.GetRamSizeReal(); + const u32 offset = Mem1ViewOffset(address); + if (ram != nullptr && offset < ram_size && ram_size - offset >= bytes) + return {ram, offset}; + u8* const exram = memory.GetEXRAM(); + if (exram == nullptr || DolphinWebHelperDisabled(DOLPHIN_WEB_DISABLE_PA_MEM2_INLINE) || + (store && (address >> 28) != 0x9)) + return {nullptr, 0}; + const u32 exram_size = memory.GetExRamSizeReal(); + const u32 offset2 = Mem2ViewOffset(address); + if (offset2 < exram_size && exram_size - offset2 >= bytes) + return {exram, offset2}; + return {nullptr, 0}; } s32 DecodeConditionalBranchDisplacement(UGeckoInstruction inst) @@ -1703,12 +1748,13 @@ const u32 element_bytes = PairedQuantizedElementBytes(type); const u32 bytes = inst.W ? element_bytes : element_bytes * 2; - const u32 offset = Mem1ViewOffset(address); - const u32 ram_size = system.GetMemory().GetRamSizeReal(); - if (offset > ram_size - bytes) + // PlayAgain 0103: MEM1, or MEM2 on a Wii (PaFastRamView). + const PaRamView view = PaFastRamView(system, address, bytes, store); + if (view.base == nullptr) return false; + const u32 offset = view.offset; - u8* ram = system.GetMemory().GetRAM(); + u8* ram = view.base; if (load) { @@ -2232,8 +2278,35 @@ u32 l1_base; u32 l1_size; u32 region_call_offset; // 0142: PowerPCState::region_call_depth (stats after it) + // PlayAgain 0103: MEM2 (Wii): its host address and size, 0 without MEM2 or with ?mem2inline=0 + // (the emitters then add no MEM2 arm: the same bytes as before), and the address of a cell that + // holds its host address. Region fragments read MEM2's base from that cell (a fixed .bss + // address, hashed in WasmLayoutHash) instead of embedding a session's heap address outside a + // heap site; blocks embed the base as they embed MEM1's. + u32 exram_base = 0; + u32 exram_size_real = 0; + u32 exram_cell = 0; }; +// PlayAgain 0103: MEM2's base for region fragments (WasmStateLayout::exram_cell). +std::atomic s_pa_exram_cell{0}; +static_assert(sizeof(std::atomic) == sizeof(u32)); + +void PaFillMem2Layout(WasmStateLayout& layout, Core::System& system) +{ + auto& memory = system.GetMemory(); + u8* const exram = memory.GetEXRAM(); + if (exram == nullptr || DolphinWebHelperDisabled(DOLPHIN_WEB_DISABLE_PA_MEM2_INLINE)) + { + layout.exram_base = layout.exram_size_real = layout.exram_cell = 0; + return; + } + layout.exram_base = static_cast(reinterpret_cast(exram)); + layout.exram_size_real = memory.GetExRamSizeReal(); + s_pa_exram_cell.store(layout.exram_base, std::memory_order_relaxed); + layout.exram_cell = static_cast(reinterpret_cast(&s_pa_exram_cell)); +} + constexpr u32 WASM_IMPORT_FASTCHECK_GPFIFO_INDEX = 14; constexpr u32 WASM_IMPORT_INTERPRET_INDEX = 13; constexpr u32 WASM_IMPORT_FMA_INDEX = 15; @@ -3562,15 +3635,92 @@ EmitLocalSet(body, 1); } +// PlayAgain 0103: which RAM the "RAM" emitters below address. MEM1 by default; MEM2 while a +// PaMem2View is alive, which emits a MEM2 arm with the MEM1 arm's own instructions on MEM2's +// view offset (Mem2ViewOffset), size and base. Only on a Wii, with ?mem2inline on +// (layout.exram_size_real != 0): otherwise no MEM2 arm and the same bytes as before. +// A store's MEM2 arm takes the cached 0x9 view only (PaFastRamView says why). +thread_local bool t_pa_mem2_view = false; +thread_local bool t_pa_mem2_store = false; +struct PaMem2View +{ + explicit PaMem2View(bool store) + { + t_pa_mem2_view = true; + t_pa_mem2_store = store; + } + ~PaMem2View() { t_pa_mem2_view = t_pa_mem2_store = false; } + PaMem2View(const PaMem2View&) = delete; + PaMem2View& operator=(const PaMem2View&) = delete; +}; + +bool PaMem2Inline(const WasmStateLayout& layout) +{ + return layout.exram_size_real != 0; +} + +u32 PaViewSizeReal(const WasmStateLayout& layout) +{ + return t_pa_mem2_view ? layout.exram_size_real : layout.ram_size_real; +} + +// [] -> [i32 host address of the view's byte 0]. MEM1: its heap site. MEM2: in a block the +// constant; in a region fragment loaded from the cell (WasmStateLayout::exram_cell). +void EmitPaViewBase(std::vector& body, const WasmStateLayout& layout) +{ + if (!t_pa_mem2_view) + { + EmitHeapBase(body, layout.ram_base, WASM_HEAP_RAM); + return; + } + if (!s_wasm_fragment_relocs) + { + EmitI32Const(body, layout.exram_base); + return; + } + EmitI32Const(body, layout.exram_cell); + body.push_back(0x28); // i32.load + body.push_back(0x02); + EmitU32Leb(body, 0); +} + +// After a MEM1 arm's `else`: `cond; if ; arm; else` again on MEM2 when the layout has it. +// Returns the `end`s it leaves open (0 or 1), for the caller's chain of ends. +template +u32 PaEmitMem2ElseArm(std::vector& body, const WasmStateLayout& layout, bool store, u8 type, + Cond cond, Arm arm) +{ + if (!PaMem2Inline(layout)) + return 0; + PaMem2View view(store); + cond(); + HintNextIf(body, true); // past MEM1, a Wii access is likely MEM2 + body.push_back(0x04); + body.push_back(type); + arm(); + body.push_back(0x05); + return 1; +} + +void PaEmitEnds(std::vector& body, u32 ends) +{ + for (u32 i = 0; i < ends; ++i) + body.push_back(0x0b); +} + // [i32 address] -> [i32 offset]: the MEM1 offset of an address in the // 0x80000000 (cached) or 0xC0000000 (uncached) view, the only ones the // standard BATs map; any other address yields a value >= 0x3fffffff, so the // caller's "offset < RAM size" test sends it to the MMU helper, as the // interpreter does (0x0xxxxxxx and 0x4xxxxxxx are not RAM). +// PlayAgain 0103: in a MEM2 arm, the MEM2 offset (Mem2ViewOffset: the 0x9 and 0xD views; a +// store's arm keeps the 0xD view's bit 30, so only the 0x9 view lands below MEM2's size). void EmitMem1ViewOffset(std::vector& body) { - EmitI32Const(body, 0x80000000); + EmitI32Const(body, t_pa_mem2_view ? 0x90000000 : 0x80000000); body.push_back(0x73); // i32.xor: the 0x8 and 0xC views lose bit 31 + if (t_pa_mem2_view && t_pa_mem2_store) + return; EmitI32Const(body, 0xbfffffff); body.push_back(0x71); // i32.and: the 0xC view folds onto 0x8; others keep bit 31 } @@ -3579,7 +3729,7 @@ { EmitLocalGet(body, 1); EmitMem1ViewOffset(body); - EmitI32Const(body, layout.ram_size_real - sizeof(u32)); + EmitI32Const(body, PaViewSizeReal(layout) - sizeof(u32)); body.push_back(0x4d); } @@ -3587,7 +3737,7 @@ { EmitLocalGet(body, 1); EmitMem1ViewOffset(body); - EmitI32Const(body, layout.ram_size_real - bytes); + EmitI32Const(body, PaViewSizeReal(layout) - bytes); body.push_back(0x4d); } @@ -3606,7 +3756,7 @@ { EmitLocalGet(body, 1); EmitMem1ViewOffset(body); - EmitI32Const(body, layout.ram_size_real - 32); + EmitI32Const(body, PaViewSizeReal(layout) - 32); body.push_back(0x4d); } @@ -3621,16 +3771,17 @@ body.push_back(0x72); } +// PlayAgain 0103: in a MEM2 arm, the low 28 bits (0x9 and 0xD views alike). void EmitRamOffset(std::vector& body) { EmitLocalGet(body, 1); - EmitI32Const(body, 0x3fffffff); + EmitI32Const(body, t_pa_mem2_view ? 0x0fffffff : 0x3fffffff); body.push_back(0x71); } void EmitRamAddress(std::vector& body, const WasmStateLayout& layout) { - EmitHeapBase(body, layout.ram_base, WASM_HEAP_RAM); + EmitPaViewBase(body, layout); EmitRamOffset(body); body.push_back(0x6a); } @@ -3855,80 +4006,96 @@ EmitU32Leb(body, byte_offset); } -void EmitReadU32MaybeDirect(std::vector& body, const WasmStateLayout& layout) +// PlayAgain 0103: [] -> [i32]: whether `bytes` from local 1 lie in the view (MEM1, or MEM2 in a +// MEM2 arm), the view offset teed into local 3. +void EmitPaViewOffsetFits(std::vector& body, const WasmStateLayout& layout, u32 bytes) { EmitLocalGet(body, 1); EmitMem1ViewOffset(body); body.push_back(0x22); EmitU32Leb(body, 3); - EmitI32Const(body, layout.ram_size_real - sizeof(u32)); + EmitI32Const(body, PaViewSizeReal(layout) - bytes); body.push_back(0x4d); +} + +// PlayAgain 0103: `fits(MEM1) if arm(MEM1) else [fits(MEM2) if arm(MEM2) else] helper end`. +template +void EmitPaMaybeDirect(std::vector& body, const WasmStateLayout& layout, u32 bytes, u8 type, + Arm arm, Helper helper) +{ + const bool store = type == 0x40; + EmitPaViewOffsetFits(body, layout, bytes); HintNextIf(body, true); // fast path in RAM body.push_back(0x04); - body.push_back(0x7f); - EmitHeapBase(body, layout.ram_base, WASM_HEAP_RAM); - EmitLocalGet(body, 3); - body.push_back(0x6a); - body.push_back(0x28); - body.push_back(0x00); - EmitU32Leb(body, 0); - EmitLocalSet(body, 2); - EmitBswap32FromLocal(body, 2); + body.push_back(type); + arm(); body.push_back(0x05); - EmitLocalGet(body, 0); - EmitLocalGet(body, 1); - EmitCallImport(body, 0); + const u32 ends = PaEmitMem2ElseArm(body, layout, store, type, + [&] { EmitPaViewOffsetFits(body, layout, bytes); }, arm); + helper(); + PaEmitEnds(body, ends); body.push_back(0x0b); } +void EmitReadU32MaybeDirect(std::vector& body, const WasmStateLayout& layout) +{ + EmitPaMaybeDirect( + body, layout, sizeof(u32), 0x7f, + [&] { + EmitPaViewBase(body, layout); + EmitLocalGet(body, 3); + body.push_back(0x6a); + body.push_back(0x28); + body.push_back(0x00); + EmitU32Leb(body, 0); + EmitLocalSet(body, 2); + EmitBswap32FromLocal(body, 2); + }, + [&] { + EmitLocalGet(body, 0); + EmitLocalGet(body, 1); + EmitCallImport(body, 0); + }); +} + void EmitReadU8MaybeDirect(std::vector& body, const WasmStateLayout& layout) { - EmitLocalGet(body, 1); - EmitMem1ViewOffset(body); - body.push_back(0x22); - EmitU32Leb(body, 3); - EmitI32Const(body, layout.ram_size_real - sizeof(u8)); - body.push_back(0x4d); - HintNextIf(body, true); // fast path in RAM - body.push_back(0x04); - body.push_back(0x7f); - EmitHeapBase(body, layout.ram_base, WASM_HEAP_RAM); - EmitLocalGet(body, 3); - body.push_back(0x6a); - body.push_back(0x2d); - body.push_back(0x00); - EmitU32Leb(body, 0); - body.push_back(0x05); - EmitLocalGet(body, 0); - EmitLocalGet(body, 1); - EmitCallImport(body, 2); - body.push_back(0x0b); + EmitPaMaybeDirect( + body, layout, sizeof(u8), 0x7f, + [&] { + EmitPaViewBase(body, layout); + EmitLocalGet(body, 3); + body.push_back(0x6a); + body.push_back(0x2d); + body.push_back(0x00); + EmitU32Leb(body, 0); + }, + [&] { + EmitLocalGet(body, 0); + EmitLocalGet(body, 1); + EmitCallImport(body, 2); + }); } void EmitReadU16MaybeDirect(std::vector& body, const WasmStateLayout& layout) { - EmitLocalGet(body, 1); - EmitMem1ViewOffset(body); - body.push_back(0x22); - EmitU32Leb(body, 3); - EmitI32Const(body, layout.ram_size_real - sizeof(u16)); - body.push_back(0x4d); - HintNextIf(body, true); // fast path in RAM - body.push_back(0x04); - body.push_back(0x7f); - EmitHeapBase(body, layout.ram_base, WASM_HEAP_RAM); - EmitLocalGet(body, 3); - body.push_back(0x6a); - body.push_back(0x2f); - body.push_back(0x00); - EmitU32Leb(body, 0); - EmitLocalSet(body, 2); - EmitBswap16FromLocal(body, 2); - body.push_back(0x05); - EmitLocalGet(body, 0); - EmitLocalGet(body, 1); - EmitCallImport(body, 4); - body.push_back(0x0b); + EmitPaMaybeDirect( + body, layout, sizeof(u16), 0x7f, + [&] { + EmitPaViewBase(body, layout); + EmitLocalGet(body, 3); + body.push_back(0x6a); + body.push_back(0x2f); + body.push_back(0x00); + EmitU32Leb(body, 0); + EmitLocalSet(body, 2); + EmitBswap16FromLocal(body, 2); + }, + [&] { + EmitLocalGet(body, 0); + EmitLocalGet(body, 1); + EmitCallImport(body, 4); + }); } void EmitSignExtendI16(std::vector& body) @@ -3947,84 +4114,69 @@ void EmitWriteU32MaybeDirect(std::vector& body, const WasmStateLayout& layout) { - EmitLocalGet(body, 1); - EmitMem1ViewOffset(body); - body.push_back(0x22); - EmitU32Leb(body, 3); - EmitI32Const(body, layout.ram_size_real - sizeof(u32)); - body.push_back(0x4d); - HintNextIf(body, true); // fast path in RAM - body.push_back(0x04); - body.push_back(0x40); - EmitBswap32FromLocal(body, 2); - EmitLocalSet(body, 2); - EmitHeapBase(body, layout.ram_base, WASM_HEAP_RAM); - EmitLocalGet(body, 3); - body.push_back(0x6a); - EmitLocalGet(body, 2); - body.push_back(0x36); - body.push_back(0x00); - EmitU32Leb(body, 0); - body.push_back(0x05); - EmitLocalGet(body, 0); - EmitLocalGet(body, 1); - EmitLocalGet(body, 2); - EmitCallImport(body, 1); - body.push_back(0x0b); + EmitPaMaybeDirect( + body, layout, sizeof(u32), 0x40, + [&] { + EmitBswap32FromLocal(body, 2); + EmitLocalSet(body, 2); + EmitPaViewBase(body, layout); + EmitLocalGet(body, 3); + body.push_back(0x6a); + EmitLocalGet(body, 2); + body.push_back(0x36); + body.push_back(0x00); + EmitU32Leb(body, 0); + }, + [&] { + EmitLocalGet(body, 0); + EmitLocalGet(body, 1); + EmitLocalGet(body, 2); + EmitCallImport(body, 1); + }); } void EmitWriteU8MaybeDirect(std::vector& body, const WasmStateLayout& layout) { - EmitLocalGet(body, 1); - EmitMem1ViewOffset(body); - body.push_back(0x22); - EmitU32Leb(body, 3); - EmitI32Const(body, layout.ram_size_real - sizeof(u8)); - body.push_back(0x4d); - HintNextIf(body, true); // fast path in RAM - body.push_back(0x04); - body.push_back(0x40); - EmitHeapBase(body, layout.ram_base, WASM_HEAP_RAM); - EmitLocalGet(body, 3); - body.push_back(0x6a); - EmitLocalGet(body, 2); - body.push_back(0x3a); - body.push_back(0x00); - EmitU32Leb(body, 0); - body.push_back(0x05); - EmitLocalGet(body, 0); - EmitLocalGet(body, 1); - EmitLocalGet(body, 2); - EmitCallImport(body, 3); - body.push_back(0x0b); + EmitPaMaybeDirect( + body, layout, sizeof(u8), 0x40, + [&] { + EmitPaViewBase(body, layout); + EmitLocalGet(body, 3); + body.push_back(0x6a); + EmitLocalGet(body, 2); + body.push_back(0x3a); + body.push_back(0x00); + EmitU32Leb(body, 0); + }, + [&] { + EmitLocalGet(body, 0); + EmitLocalGet(body, 1); + EmitLocalGet(body, 2); + EmitCallImport(body, 3); + }); } void EmitWriteU16MaybeDirect(std::vector& body, const WasmStateLayout& layout) { - EmitLocalGet(body, 1); - EmitMem1ViewOffset(body); - body.push_back(0x22); - EmitU32Leb(body, 3); - EmitI32Const(body, layout.ram_size_real - sizeof(u16)); - body.push_back(0x4d); - HintNextIf(body, true); // fast path in RAM - body.push_back(0x04); - body.push_back(0x40); - EmitBswap16FromLocal(body, 2); - EmitLocalSet(body, 2); - EmitHeapBase(body, layout.ram_base, WASM_HEAP_RAM); - EmitLocalGet(body, 3); - body.push_back(0x6a); - EmitLocalGet(body, 2); - body.push_back(0x3b); - body.push_back(0x00); - EmitU32Leb(body, 0); - body.push_back(0x05); - EmitLocalGet(body, 0); - EmitLocalGet(body, 1); - EmitLocalGet(body, 2); - EmitCallImport(body, 5); - body.push_back(0x0b); + EmitPaMaybeDirect( + body, layout, sizeof(u16), 0x40, + [&] { + EmitBswap16FromLocal(body, 2); + EmitLocalSet(body, 2); + EmitPaViewBase(body, layout); + EmitLocalGet(body, 3); + body.push_back(0x6a); + EmitLocalGet(body, 2); + body.push_back(0x3b); + body.push_back(0x00); + EmitU32Leb(body, 0); + }, + [&] { + EmitLocalGet(body, 0); + EmitLocalGet(body, 1); + EmitLocalGet(body, 2); + EmitCallImport(body, 5); + }); } void EmitImportedHaltCheck(std::vector& body) @@ -5682,6 +5834,15 @@ EmitDirectStfs(body, layout, inst, false); body.push_back(0x05); + // PlayAgain 0103: the same arm on MEM2. + const u32 mem2_ends = PaEmitMem2ElseArm( + body, layout, !load, 0x40, [&] { EmitAlignedFloatFastPathCondition(body, layout, false); }, + [&] { + if (load) + EmitDirectLfs(body, layout, inst, false); + else + EmitDirectStfs(body, layout, inst, false); + }); EmitAlignedFloatFastPathCondition(body, layout, true); body.push_back(0x04); body.push_back(0x40); @@ -5706,6 +5867,7 @@ body.push_back(0x0b); if (!load) body.push_back(0x0b); + PaEmitEnds(body, mem2_ends); body.push_back(0x0b); if (update) @@ -5739,6 +5901,10 @@ EmitDirectLfdFromRam(body, layout, inst); body.push_back(0x05); + // PlayAgain 0103: the same arm on MEM2. + const u32 mem2_ends = PaEmitMem2ElseArm( + body, layout, false, 0x40, [&] { EmitHighMem1FloatCondition(body, layout, sizeof(u64)); }, + [&] { EmitDirectLfdFromRam(body, layout, inst); }); EmitAlignedFloatFastPathCondition(body, layout, true, sizeof(u64)); body.push_back(0x04); body.push_back(0x40); @@ -5747,6 +5913,7 @@ body.push_back(0x05); EmitLfdImportCall(body, inst, op.address, downcount); body.push_back(0x0b); + PaEmitEnds(body, mem2_ends); body.push_back(0x0b); if (update) @@ -5769,6 +5936,10 @@ EmitDirectStfdToRam(body, layout, inst); body.push_back(0x05); + // PlayAgain 0103: the same arm on MEM2. + const u32 mem2_ends = PaEmitMem2ElseArm( + body, layout, true, 0x40, [&] { EmitHighMem1FloatCondition(body, layout, sizeof(u64)); }, + [&] { EmitDirectStfdToRam(body, layout, inst); }); EmitAlignedFloatFastPathCondition(body, layout, true, sizeof(u64)); body.push_back(0x04); body.push_back(0x40); @@ -5777,6 +5948,7 @@ body.push_back(0x05); EmitFpCall(body, op, downcount); body.push_back(0x0b); + PaEmitEnds(body, mem2_ends); body.push_back(0x0b); if (update) @@ -5794,41 +5966,49 @@ const u32 first_reg = load ? inst.RD : inst.RS; const u32 bytes = (32 - first_reg) * sizeof(u32); - EmitStateLoadU32(body, layout.msr_offset); - EmitI32Const(body, 1); - body.push_back(0x71); - body.push_back(0x45); + const auto cond = [&] { + EmitStateLoadU32(body, layout.msr_offset); + EmitI32Const(body, 1); + body.push_back(0x71); + body.push_back(0x45); - EmitLocalGet(body, 1); - EmitI32Const(body, 3); - body.push_back(0x71); - body.push_back(0x45); - body.push_back(0x71); + EmitLocalGet(body, 1); + EmitI32Const(body, 3); + body.push_back(0x71); + body.push_back(0x45); + body.push_back(0x71); - EmitIsMem1AddressForSize(body, layout, bytes); - body.push_back(0x71); - + EmitIsMem1AddressForSize(body, layout, bytes); + body.push_back(0x71); + }; + const auto arm = [&] { + for (u32 reg = first_reg; reg <= 31; ++reg) + { + const u32 byte_offset = (reg - first_reg) * sizeof(u32); + if (load) + { + EmitStateStoreU32Prefix(body); + EmitLoadU32BigEndianFromRamAddressOffset(body, layout, byte_offset); + EmitStateStoreU32Suffix(body, GprOffset(layout, reg)); + } + else + { + EmitStateLoadU32(body, GprOffset(layout, reg)); + EmitLocalSet(body, 2); + EmitStoreU32BigEndianToRamAddressOffset(body, layout, byte_offset); + } + } + }; + cond(); HintNextIf(body, true); // aligned, in RAM body.push_back(0x04); body.push_back(0x40); - for (u32 reg = first_reg; reg <= 31; ++reg) - { - const u32 byte_offset = (reg - first_reg) * sizeof(u32); - if (load) - { - EmitStateStoreU32Prefix(body); - EmitLoadU32BigEndianFromRamAddressOffset(body, layout, byte_offset); - EmitStateStoreU32Suffix(body, GprOffset(layout, reg)); - } - else - { - EmitStateLoadU32(body, GprOffset(layout, reg)); - EmitLocalSet(body, 2); - EmitStoreU32BigEndianToRamAddressOffset(body, layout, byte_offset); - } - } + arm(); body.push_back(0x05); + // PlayAgain 0103: the same arm on MEM2. + const u32 mem2_ends = PaEmitMem2ElseArm(body, layout, !load, 0x40, cond, arm); EmitSystemCall(body, op, downcount); + PaEmitEnds(body, mem2_ends); body.push_back(0x0b); } @@ -5855,6 +6035,10 @@ EmitDirectStfiwx(body, layout, inst, false); body.push_back(0x05); + // PlayAgain 0103: the same arm on MEM2. + const u32 mem2_ends = PaEmitMem2ElseArm( + body, layout, true, 0x40, [&] { EmitAlignedFloatFastPathCondition(body, layout, false); }, + [&] { EmitDirectStfiwx(body, layout, inst, false); }); EmitAlignedFloatFastPathCondition(body, layout, true); body.push_back(0x04); body.push_back(0x40); @@ -5863,6 +6047,7 @@ body.push_back(0x05); EmitFpCall(body, op, downcount); body.push_back(0x0b); + PaEmitEnds(body, mem2_ends); body.push_back(0x0b); } @@ -6630,49 +6815,55 @@ EmitEffectiveAddress12(body, layout, inst, update); EmitLocalSet(body, 1); - EmitPsqFloatFastPathCondition(body, layout, inst, !load); - body.push_back(0x04); - body.push_back(0x40); - - // psq SIMD: both lanes at once (off with DISABLE_NATIVE_PS, like 0143). - const bool pair = !inst.W && !DolphinWebHelperDisabled(DOLPHIN_WEB_DISABLE_NATIVE_PS); - if (load && pair) - { - EmitLoadPsqFloatPair(body, layout, inst.RD); - } - else if (!load && pair) - { - EmitStorePsqFloatPair(body, layout, inst.RS); - } - else if (load) - { - EmitLoadPsqFloatLane(body, layout, inst.RD, 0, 0); - if (inst.W) + // PlayAgain 0103: the float arm as a lambda, emitted on MEM1 and then on MEM2. + const auto float_arm = [&] { + // psq SIMD: both lanes at once (off with DISABLE_NATIVE_PS, like 0143). + const bool pair = !inst.W && !DolphinWebHelperDisabled(DOLPHIN_WEB_DISABLE_NATIVE_PS); + if (load && pair) { - EmitStateStoreU32Prefix(body); - EmitF64ConstBits(body, UINT64_C(0x3ff0000000000000)); - EmitStateStoreF64Suffix(body, PsOffset(layout, inst.RD, 1)); + EmitLoadPsqFloatPair(body, layout, inst.RD); } + else if (!load && pair) + { + EmitStorePsqFloatPair(body, layout, inst.RS); + } + else if (load) + { + EmitLoadPsqFloatLane(body, layout, inst.RD, 0, 0); + if (inst.W) + { + EmitStateStoreU32Prefix(body); + EmitF64ConstBits(body, UINT64_C(0x3ff0000000000000)); + EmitStateStoreF64Suffix(body, PsOffset(layout, inst.RD, 1)); + } + else + { + EmitLoadPsqFloatLane(body, layout, inst.RD, 1, sizeof(u32)); + } + } else { - EmitLoadPsqFloatLane(body, layout, inst.RD, 1, sizeof(u32)); + EmitStorePsqFloatLane(body, layout, inst.RS, 0, 0); + if (!inst.W) + EmitStorePsqFloatLane(body, layout, inst.RS, 1, sizeof(u32)); } - } - else - { - EmitStorePsqFloatLane(body, layout, inst.RS, 0, 0); - if (!inst.W) - EmitStorePsqFloatLane(body, layout, inst.RS, 1, sizeof(u32)); - } - if (update) - { - EmitStateStoreU32Prefix(body); - EmitLocalGet(body, 1); - EmitStateStoreU32Suffix(body, GprOffset(layout, inst.RA)); - } + if (update) + { + EmitStateStoreU32Prefix(body); + EmitLocalGet(body, 1); + EmitStateStoreU32Suffix(body, GprOffset(layout, inst.RA)); + } + }; + EmitPsqFloatFastPathCondition(body, layout, inst, !load); + body.push_back(0x04); + body.push_back(0x40); + float_arm(); body.push_back(0x05); + const u32 mem2_ends = PaEmitMem2ElseArm( + body, layout, !load, 0x40, [&] { EmitPsqFloatFastPathCondition(body, layout, inst, !load); }, + float_arm); if (!load) { EmitPsqGatherPipeFloatStoreCondition(body, layout, inst); @@ -6724,6 +6915,7 @@ body.push_back(0x0b); if (!load) body.push_back(0x0b); + PaEmitEnds(body, mem2_ends); body.push_back(0x0b); } @@ -6736,27 +6928,34 @@ body.push_back(0x71); EmitLocalSet(body, 1); - EmitStateLoadU32(body, SprOffset(layout, SPR_HID0)); - EmitI32Const(body, 1u << 14); - body.push_back(0x71); - EmitI32Const(body, 0); - body.push_back(0x47); - EmitStateLoadU8(body, layout.dcache_enabled_offset); - body.push_back(0x45); - body.push_back(0x71); - EmitIsMem1LineAddress(body, layout); - body.push_back(0x71); - EmitIsOutsideLowDcbzHackRange(body); - body.push_back(0x71); + const auto cond = [&] { + EmitStateLoadU32(body, SprOffset(layout, SPR_HID0)); + EmitI32Const(body, 1u << 14); + body.push_back(0x71); + EmitI32Const(body, 0); + body.push_back(0x47); + EmitStateLoadU8(body, layout.dcache_enabled_offset); + body.push_back(0x45); + body.push_back(0x71); + EmitIsMem1LineAddress(body, layout); + body.push_back(0x71); + EmitIsOutsideLowDcbzHackRange(body); + body.push_back(0x71); + }; + const auto arm = [&] { + for (u32 offset = 0; offset < 32; offset += sizeof(u32)) + EmitStoreZeroU32ToRamAddress(body, layout, offset); + }; + cond(); HintNextIf(body, true); // a line in RAM body.push_back(0x04); body.push_back(0x40); - - for (u32 offset = 0; offset < 32; offset += sizeof(u32)) - EmitStoreZeroU32ToRamAddress(body, layout, offset); - + arm(); body.push_back(0x05); + // PlayAgain 0103: the same arm on MEM2. + const u32 mem2_ends = PaEmitMem2ElseArm(body, layout, true, 0x40, cond, arm); EmitSystemCall(body, op, downcount); + PaEmitEnds(body, mem2_ends); body.push_back(0x0b); } @@ -11912,11 +12111,11 @@ ((update || inst.RA) ? ppc_state.gpr[inst.RA] + u32(inst.SIMM_16) : u32(inst.SIMM_16)); - u8* ram = system.GetMemory().GetRAM(); - const u32 offset = Mem1ViewOffset(address); - const u32 ram_size = system.GetMemory().GetRamSizeReal(); - const bool in_ram = !ppc_state.msr.LE && ram != nullptr && offset < ram_size && - ram_size - offset >= sizeof(u32); + // PlayAgain 0103: MEM1, or MEM2 on a Wii (PaFastRamView). + const PaRamView view = PaFastRamView(system, address, sizeof(u32), store); + u8* ram = view.base; + const u32 offset = view.offset; + const bool in_ram = !ppc_state.msr.LE && ram != nullptr; if (load) { @@ -11989,12 +12188,12 @@ ((update || inst.RA) ? ppc_state.gpr[inst.RA] + u32(inst.SIMM_16) : u32(inst.SIMM_16)); - u8* ram = system.GetMemory().GetRAM(); - const u32 offset = Mem1ViewOffset(address); - const u32 ram_size = system.GetMemory().GetRamSizeReal(); const u32 bytes = (load_half || store_half) ? sizeof(u16) : sizeof(u8); - const bool in_ram = !ppc_state.msr.LE && ram != nullptr && offset < ram_size && - ram_size - offset >= bytes; + // PlayAgain 0103: MEM1, or MEM2 on a Wii (PaFastRamView). + const PaRamView view = PaFastRamView(system, address, bytes, store_byte || store_half); + u8* ram = view.base; + const u32 offset = view.offset; + const bool in_ram = !ppc_state.msr.LE && ram != nullptr; if (load_byte) { @@ -12061,11 +12260,11 @@ inst.RA ? ppc_state.gpr[inst.RA] + u32(inst.SIMM_16) : u32(inst.SIMM_16); const u32 register_count = 32 - inst.RD; const u32 bytes = register_count * sizeof(u32); - u8* ram = system.GetMemory().GetRAM(); - const u32 offset = Mem1ViewOffset(start_address); - const u32 ram_size = system.GetMemory().GetRamSizeReal(); - const bool in_ram = !ppc_state.msr.LE && ram != nullptr && offset < ram_size && - ram_size - offset >= bytes; + // PlayAgain 0103: MEM1, or MEM2 on a Wii (PaFastRamView). + const PaRamView view = PaFastRamView(system, start_address, bytes, store); + u8* ram = view.base; + const u32 offset = view.offset; + const bool in_ram = !ppc_state.msr.LE && ram != nullptr; if (load) { @@ -12426,12 +12625,12 @@ return true; } - u8* ram = system.GetMemory().GetRAM(); - const u32 offset = Mem1ViewOffset(address); - const u32 ram_size = system.GetMemory().GetRamSizeReal(); - const u32 bytes = load_double ? sizeof(u64) : sizeof(u32); - const bool in_ram = - !ppc_state.msr.LE && ram != nullptr && offset < ram_size && ram_size - offset >= bytes; + const u32 bytes = load_double ? sizeof(u64) : sizeof(u32); + // PlayAgain 0103: MEM1, or MEM2 on a Wii (PaFastRamView). + const PaRamView view = PaFastRamView(system, address, bytes, store); + u8* ram = view.base; + const u32 offset = view.offset; + const bool in_ram = !ppc_state.msr.LE && ram != nullptr; if (load_single) { @@ -13185,7 +13384,7 @@ !did_merge && prefix_count > 0 && m_code_buffer[prefix_count - 1].canEndBlock; const auto state_base = reinterpret_cast(&m_ppc_state); - const WasmStateLayout layout = { + WasmStateLayout layout = { static_cast(reinterpret_cast(&m_ppc_state.npc) - state_base), static_cast(reinterpret_cast(m_ppc_state.gpr) - state_base), static_cast(reinterpret_cast(m_ppc_state.ps) - state_base), @@ -13211,6 +13410,7 @@ static_cast(reinterpret_cast(m_system.GetMemory().GetL1Cache())), m_system.GetMemory().GetL1CacheSize(), }; + PaFillMem2Layout(layout, m_system); // PlayAgain 0103 const bool write_npc = did_merge ? merge_write_npc : (compiled_entire_block && !compiled_block_ends_with_branch); const std::span ops = @@ -14570,6 +14770,45 @@ // that travel in the state words: update forms drift their base around it. constexpr u32 JIT_ORACLE_WINDOW_VA = JIT_ORACLE_SCRATCH_VA - 0x800; constexpr u32 JIT_ORACLE_WINDOW_BYTES = 0x1000; + +// PlayAgain 0103: the oracle on a Wii's memory (MEM2 present, mapped by the standard BATs: +// 0x90000000 cached and 0xD0000000 uncached onto physical 0x10000000), its scratch bytes (state +// words 176-239) and compared window in MEM1 (mode 1), in MEM2's middle (2), at MEM2's last 256 +// bytes (3) or at its first 256 (4). Mode 0: GameCube memory, as before. Once on a Wii's memory +// the oracle stays there (mode 0 is refused then). +u32 s_pa_oracle_mem2 = 0; +struct PaOracleArea +{ + u32 scratch_va; + u8* scratch; + u32 window_va; + u8* window; +}; +PaOracleArea PaOracleAreaNow(Core::System& system) +{ + auto& memory = system.GetMemory(); + u8* const ram = memory.GetRAM(); + u8* const exram = memory.GetEXRAM(); + const u32 size2 = memory.GetExRamSizeReal(); + if (exram != nullptr) + { + switch (s_pa_oracle_mem2) + { + case 2: + return {0x90100000, exram + 0x00100000, 0x900ff800, exram + 0x000ff800}; + case 3: + return {0x90000000 + size2 - 0x100, exram + size2 - 0x100, + 0x90000000 + size2 - JIT_ORACLE_WINDOW_BYTES, exram + size2 - JIT_ORACLE_WINDOW_BYTES}; + case 4: + return {0x90000000, exram, 0x90000000, exram}; + default: + break; + } + } + return {JIT_ORACLE_SCRATCH_VA, ram + (JIT_ORACLE_SCRATCH_VA & 0x01ffffff), JIT_ORACLE_WINDOW_VA, + ram + (JIT_ORACLE_WINDOW_VA & 0x01ffffff)}; +} +std::array s_pa_oracle_window_saved{}; struct JitOracleBuffer { @@ -14614,8 +14853,7 @@ const u32* w = &in[48 + i * 4]; ppc.ps[i].SetBoth(u64{w[0]} | (u64{w[1]} << 32), u64{w[2]} | (u64{w[3]} << 32)); } - std::memcpy(system.GetMemory().GetRAM() + (JIT_ORACLE_SCRATCH_VA & 0x01ffffff), &in[176], - JIT_ORACLE_SCRATCH_BYTES); + std::memcpy(PaOracleAreaNow(system).scratch, &in[176], JIT_ORACLE_SCRATCH_BYTES); SRR0(ppc) = in[240]; SRR1(ppc) = in[241]; ppc.spr[SPR_DAR] = in[242]; @@ -14647,8 +14885,7 @@ w[2] = static_cast(b); w[3] = static_cast(b >> 32); } - std::memcpy(&out[176], system.GetMemory().GetRAM() + (JIT_ORACLE_SCRATCH_VA & 0x01ffffff), - JIT_ORACLE_SCRATCH_BYTES); + std::memcpy(&out[176], PaOracleAreaNow(system).scratch, JIT_ORACLE_SCRATCH_BYTES); out[240] = SRR0(ppc); out[241] = SRR1(ppc); out[242] = ppc.spr[SPR_DAR]; @@ -14661,6 +14898,13 @@ // The interpreter's FPSCR writes assert they run on the CPU thread. Core::DeclareAsCPUThread(); auto& memory = system.GetMemory(); + // PlayAgain 0103: a Wii's memory for the MEM2 modes. + if (s_pa_oracle_mem2 != 0 && !system.IsWii()) + { + if (memory.GetRAM()) + memory.Shutdown(); + system.SetIsWii(true); + } if (!memory.GetRAM()) memory.Init(); if (!memory.GetRAM()) @@ -14675,6 +14919,14 @@ } ppc.spr[SPR_DBAT1U] = 0xc0001fff; ppc.spr[SPR_DBAT1U + 1] = 0x0000002a; + if (system.IsWii()) + { + // PlayAgain 0103: MEM2's views as a Wii game's BATs leave them. + ppc.spr[SPR_DBAT2U] = 0x90001fff; + ppc.spr[SPR_DBAT2U + 1] = 0x10000002; + ppc.spr[SPR_DBAT3U] = 0xd0001fff; + ppc.spr[SPR_DBAT3U + 1] = 0x1000002a; + } ppc.spr[SPR_HID2] = 0xa0000000; HID0(ppc).DCE = 1; // as libogc leaves it; dcbz raises an alignment exception otherwise system.GetMMU().DBATUpdated(); @@ -14809,7 +15061,8 @@ } system.GetPowerPC().GetBreakPoints().Clear(); auto& ppc = system.GetPPCState(); - u8* const window = memory.GetRAM() + (JIT_ORACLE_WINDOW_VA & 0x01ffffff); + const PaOracleArea area = PaOracleAreaNow(system); // PlayAgain 0103 + u8* const window = area.window; JitOracleLoadState(system, s_jit_oracle.input); std::memcpy(s_jit_oracle_window_start.data(), window, JIT_ORACLE_WINDOW_BYTES); @@ -14832,7 +15085,7 @@ const auto offset = [&](const void* field) { return static_cast(reinterpret_cast(field) - state_base); }; - const WasmStateLayout layout = { + WasmStateLayout layout = { offset(&ppc.npc), offset(ppc.gpr), offset(ppc.ps), offset(ppc.cr.fields), offset(ppc.spr), offset(&ppc.msr), offset(&ppc.fpscr), offset(&ppc.feature_flags), offset(&ppc.pagetable_update_pending), offset(&ppc.xer_ca), offset(&ppc.xer_so_ov), @@ -14844,6 +15097,7 @@ memory.GetL1CacheSize(), offset(&ppc.region_call_depth), }; + PaFillMem2Layout(layout, system); // PlayAgain 0103 int result = 0; while (budget && ppc.pc != JIT_ORACLE_SENTINEL_VA && (ppc.pc - JIT_ORACLE_CODE_VA) < word_count * 4) @@ -14924,7 +15178,7 @@ if (window[i] != s_jit_oracle_window_interp[i]) { s_jit_oracle_message = fmt::format("memory at {:#010x}: interp {:#04x} jit {:#04x}", - JIT_ORACLE_WINDOW_VA + i, s_jit_oracle_window_interp[i], + area.window_va + i, s_jit_oracle_window_interp[i], window[i]); return 1; } @@ -14932,6 +15186,30 @@ return std::memcmp(s_jit_oracle.interp, s_jit_oracle.jit, sizeof s_jit_oracle.jit) == 0 ? 0 : 1; } +// PlayAgain 0103: the oracle's memory mode (s_pa_oracle_mem2); returns the scratch bytes' +// address, 0 when refused. +extern "C" EMSCRIPTEN_KEEPALIVE u32 DolphinWeb_PaOracleMem2ForTest(u32 mode) +{ + auto& system = Core::System::GetInstance(); + if (mode > 4 || (mode == 0 && system.IsWii())) + return 0; + s_pa_oracle_mem2 = mode; + if (!JitOracleSetUp(system) || (mode > 1 && system.GetMemory().GetEXRAM() == nullptr)) + return 0; + return PaOracleAreaNow(system).scratch_va; +} + +// PlayAgain 0103: saves (0) or restores (1) the oracle's window, for the region oracle (whose +// harness restores MEM1 itself). +extern "C" EMSCRIPTEN_KEEPALIVE void DolphinWeb_PaOracleWindowForTest(u32 op) +{ + u8* const window = PaOracleAreaNow(Core::System::GetInstance()).window; + if (op == 0) + std::memcpy(s_pa_oracle_window_saved.data(), window, JIT_ORACLE_WINDOW_BYTES); + else + std::memcpy(window, s_pa_oracle_window_saved.data(), JIT_ORACLE_WINDOW_BYTES); +} + // Marker for tests: the block JIT's mtmsr takes pending exceptions (0076). extern "C" EMSCRIPTEN_KEEPALIVE int DolphinWeb_JitMtmsrTakesPendingExceptions() { @@ -14967,6 +15245,7 @@ std::memcpy(&layout, shifted.data(), sizeof(layout)); layout.ram_base = 0; layout.l1_base = 0; + layout.exram_base = 0; // PlayAgain 0103: a session's address, as RAM's (the cell is hashed) static_assert(sizeof(WasmStateLayout) % sizeof(u32) == 0); std::array words; std::memcpy(words.data(), &layout, sizeof(layout)); @@ -15012,7 +15291,7 @@ const auto offset = [&](const void* field) { return static_cast(reinterpret_cast(field) - state_base); }; - return { + WasmStateLayout layout = { offset(&ppc.npc), offset(ppc.gpr), offset(ppc.ps), offset(ppc.cr.fields), offset(ppc.spr), offset(&ppc.msr), offset(&ppc.fpscr), offset(&ppc.feature_flags), offset(&ppc.pagetable_update_pending), offset(&ppc.xer_ca), offset(&ppc.xer_so_ov), @@ -15024,6 +15303,8 @@ memory.GetL1CacheSize(), offset(&ppc.region_call_depth), }; + PaFillMem2Layout(layout, system); // PlayAgain 0103 + return layout; } // A block of a fragment: its expression (the block body without locals and