diff --git a/.github/instructions/raw-ne-patch-conversion.instructions.md b/.github/instructions/raw-ne-patch-conversion.instructions.md new file mode 100644 index 0000000..c5c9f31 --- /dev/null +++ b/.github/instructions/raw-ne-patch-conversion.instructions.md @@ -0,0 +1,109 @@ +--- +description: 'Workflow for converting live Ghidra NE patch experiments into raw executable patcher definitions' +applyTo: '**/patch_*.ps1, **/patch_*.md, **/docs/**/*.md' +--- + +# Raw NE Patch Conversion + +Use these instructions when a live Ghidra byte-patch experiment needs to become a reusable raw executable patch in a PowerShell patcher. + +## Goal + +- Treat the live Ghidra patch as the prototype, not the shipping artifact. +- Convert the final behavior into raw file-byte edits and NE relocation edits that a patcher script can apply and restore deterministically. +- Do not depend on the Ghidra export processor to preserve byte edits unless that exact processor path has already been verified on the target binary. + +## Required Capture From Ghidra + +Before writing the patcher definition, capture all of these from the live session: + +- Final target program name and whether it was a writable copy. +- Every patched selector address and the final intended behavior at that site. +- Exact original bytes and exact patched bytes for any plain code/data window. +- Every far call or far jump target change, including both the source address and the final target selector:offset. +- Any overwritten helper window boundaries, especially when the patch replaces existing functions instead of using empty space. +- Any branch-family coverage requirements, such as seasonal paths or alternate message lanes. + +If the patch needs to be restorable, also capture the exact original bytes and the exact original relocation-record payloads. + +## NE Conversion Rules + +### Selector To Segment Index + +For these Crusader NE executables, convert a selector-style code address to the NE segment index with: + +```text +segment_index = ((selector - 0x1000) / 8) + 1 +``` + +Use the executable's NE header to resolve the segment table entry, alignment shift, segment file offset, segment length, and relocation table location. + +### Raw Code/Data Bytes + +For a plain byte rewrite inside one code segment: + +```text +raw_file_offset = segment_file_offset + segment_relative_offset +``` + +Use the exact original byte window for verification and restore. + +### Relocation-Bearing Far Calls + +For NE `CALLF` and other relocation-backed far operands: + +- Patch the relocation record, not the placeholder immediate bytes in the opcode. +- The relocation source offset is the operand location, which starts one byte after the `9A` opcode. +- Keep the opcode bytes as `9A FF FF 00 00` unless the instruction itself is being replaced. +- Restore must write the retail relocation payload back verbatim. + +### Overwritten Windows With Existing Relocations + +If you overwrite a helper window that already contains relocation-backed far calls: + +- Enumerate every relocation record whose source offset falls inside the overwritten range. +- Either preserve that source offset as a far-call slot in the new helper or retarget it explicitly. +- Never leave a stale relocation entry attached to bytes that are no longer a relocation-bearing instruction operand. +- Prefer reusing existing relocation slots over inventing new ones. + +This is the safest way to convert a live in-memory proof patch into a raw NE patch without rewriting relocation tables structurally. + +## Patcher-Script Rules + +- Define each raw patch site with `Offset`, `Original`, and `Enabled` bytes. +- For large helper windows, keep the full original byte block in the script so restore is exact. +- For relocation-record edits, store the full 8-byte record image, not only the target segment/offset words. +- Group related sites into one named patch family with one status function and one apply/restore function. +- Status must accept both retail bytes and fully patched bytes. Any other state is unknown and must block writes. +- Restore must always write the complete retail state for the whole family, even if an older or partial patch was enabled first. + +## Validation Checklist + +After converting a Ghidra patch into a raw patcher definition: + +1. Verify the raw offsets against the current retail executable. +2. Verify every `Original` byte block matches the live unpatched file. +3. Apply the patch through the script, then re-read every site and confirm the `Enabled` bytes match exactly. +4. Confirm all overlapping relocation-backed callsites decode to the intended targets. +5. If the patch has alternate trigger branches, test each one or document the untested branch explicitly. +6. Keep the runtime test order staged so failures isolate bootstrap, UI open, and seeded-runtime behavior separately. + +## Documentation Requirements + +When a patch graduates from Ghidra prototype to raw patcher support: + +- Update the working note with the final raw offsets and what each site does. +- Record any helper-window overwrite boundaries and reused relocation slots. +- Record any export limitation that forced the raw-patch conversion. +- If MCP friction caused extra work, append a concise entry to the wishlist with the missing capability and the manual fallback. + +## Example Pattern + +The Regret `debug menu 2.0` patch is the reference model for this workflow: + +- one raw helper-window rewrite inside segment `13f8` +- one wrapper relocation retarget into that helper +- two helper relocation retargets that reuse the overwritten window's existing far-call slots +- multiple `loosecannon` fixup retargets so all trigger branches land in the debugger modal path + +Use that pattern when a live Ghidra proof patch needs to become a stable, restorable patcher option. diff --git a/Crusader.rep/idata/01/~0000001b.db/change.data.gbf b/Crusader.rep/idata/01/~0000001b.db/change.data.gbf index 7730ed0..5daeace 100644 Binary files a/Crusader.rep/idata/01/~0000001b.db/change.data.gbf and b/Crusader.rep/idata/01/~0000001b.db/change.data.gbf differ diff --git a/Crusader.rep/idata/01/~0000001b.db/change.map.gbf b/Crusader.rep/idata/01/~0000001b.db/change.map.gbf index 3cc354d..ce48377 100644 Binary files a/Crusader.rep/idata/01/~0000001b.db/change.map.gbf and b/Crusader.rep/idata/01/~0000001b.db/change.map.gbf differ diff --git a/Crusader.rep/idata/01/~0000001b.db/db.61.gbf b/Crusader.rep/idata/01/~0000001b.db/db.65.gbf similarity index 97% rename from Crusader.rep/idata/01/~0000001b.db/db.61.gbf rename to Crusader.rep/idata/01/~0000001b.db/db.65.gbf index 088a77a..39526f3 100644 Binary files a/Crusader.rep/idata/01/~0000001b.db/db.61.gbf and b/Crusader.rep/idata/01/~0000001b.db/db.65.gbf differ diff --git a/Crusader.rep/idata/01/~0000001b.db/db.60.gbf b/Crusader.rep/idata/01/~0000001b.db/db.66.gbf similarity index 96% rename from Crusader.rep/idata/01/~0000001b.db/db.60.gbf rename to Crusader.rep/idata/01/~0000001b.db/db.66.gbf index d4d51c0..f632e1e 100644 Binary files a/Crusader.rep/idata/01/~0000001b.db/db.60.gbf and b/Crusader.rep/idata/01/~0000001b.db/db.66.gbf differ diff --git a/Crusader.rep/projectState b/Crusader.rep/projectState index 1aea38c..db2f0ca 100644 --- a/Crusader.rep/projectState +++ b/Crusader.rep/projectState @@ -3,7 +3,6 @@ - @@ -12,7 +11,1179 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Crusader.rep/user/00/~0000000d.db/db.8.gbf b/Crusader.rep/user/00/~0000000d.db/db.10.gbf similarity index 99% rename from Crusader.rep/user/00/~0000000d.db/db.8.gbf rename to Crusader.rep/user/00/~0000000d.db/db.10.gbf index 5f7bfdb..e78bbc3 100644 Binary files a/Crusader.rep/user/00/~0000000d.db/db.8.gbf and b/Crusader.rep/user/00/~0000000d.db/db.10.gbf differ diff --git a/plan-mid.md b/plan-mid.md index e270b5f..577a970 100644 --- a/plan-mid.md +++ b/plan-mid.md @@ -15,6 +15,36 @@ Detailed completed analysis belongs in the files under `docs/`, not in this plan ## Progress Snapshot +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 focused live `SLUS_002.68` late art-bank corridor pass centered on `wdl_resource_bundle_load_by_index` (`0x80039444`), the header-only write sites `0x8003977c/0x80039a64`, `psx_install_type_art_active_header_and_built_resource` (`0x80045ffc`), `psx_create_image_resource_from_descriptor` (`0x80044434`), and constructor fast paths `0x80024b0c/0x80025004`. Current best read is now exporter-critical and more exact than the older “one late descriptor bank” shorthand: each WDL pass contributes two art-facing late sections, the later `8`-byte header-only override is what leaves raw `0x58`-byte active headers in `DAT_800758d8`, and constructors reuse `DAT_800758c8[type]` when that raw-header signature is present instead of rebuilding. Practical consequence is that standalone parsing should target the late header-only override stream first and treat the earlier built-resource art-install blob as a separate, still-partially-unresolved feed rather than flattening both into one guessed art bank. + +Latest verified batch: [docs/psx/map-storage-model.md](docs/psx/map-storage-model.md) now includes a 2026-04-13 live subordinate-section pass on active `SLUS_002.68` centered on `psx_apply_deferred_control_command` and `psx_control_assign_opcode_stream_by_index`. Current best read is now narrower and exporter-relevant: `DAT_80067938` provides constructor-placement-adjacent index data, `DAT_80067838` backs `8`-byte deferred-control row chains consumed by root/live-object mutation helpers, and `DAT_80067840` is an opcode-stream pointer table rather than hidden geometry. Practical consequence is that `post_audio_region_02` should be treated as a mixed resource/control payload zone until smaller typed sub-lanes are split out, not as a presumed flat floor table. + +Latest verified batch: standalone `psx-map-exporter` probe binding is now raw-only again after removing the remaining viewer-side atlas fallback from `src/export-map.js`. Fresh `LSET1_L0_probe_raw_research.json` now reports `artBindingSource = raw-typeword-bundle-slot`, and neither `viewer-reference-atlas` nor bundle `0x00085c40` appears in the regenerated probe item list. Region-02 example export also now carries structured `previewRows` in JSON alongside the false-color image so the next pass can inspect row-shaped data instead of relying on the rainbow preview alone. + +Latest verified batch: standalone `psx-map-exporter` probe rendering now excludes `section0_dispatch_roots` types `0x0042` and `0x0049` as non-map-facing portrait/talk assets. This is directly aligned with earlier `docs/psx/psx.md` negative evidence for those root families, and the fresh `LSET1_L0_probe_raw_research` rebuild dropped from `59` to `57` rendered items while keeping the skipped records visible in JSON diagnostics. Practical consequence is narrower: the removed portrait block (`55/58`) was not a palette problem, while the remaining bad colors are concentrated in unresolved mode-2 palette families (`0x0048`, `0x004b`, `0x0059`) rather than in the disproven portrait/talk fallback lane. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 focused live `SLUS_002.68` source-record palette-token provenance pass centered on constructor source-pointer stores (`0x80024b50`, `0x80025048`), section0 constructor-placement stride (`0x800258cc`), and main-visible token read split (`0x80041458`). Current best read is now exporter-decisive for region00 handling: constructors persist authored source pointer at `obj+0xa0`; main-visible token injection reads source high byte from `+0x06` for `0x003e..0x00ab` and `+0x0c` for `>=0x00ac`; and region00-style 12-byte records are therefore valid token carriers for current visible unresolved families (`0x0042`, `0x0049`, `0x0055..0x0063`) because they stay in the `<0x00ac` band. Live artifacts in this batch are one conservative helper rename (`0x80027f38`) and three targeted decompiler comments at `0x80024b50`, `0x80025048`, and `0x8004156c`. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 focused live `SLUS_002.68` main-visible palette-token submit normalization pass centered on `0x80041458` with wrapper context at `0x80031f0c/0x80041378` and submitter override gates at `0x80044e10/0x80044eb8`. Current best read is now bit-exact for exporter logic: main-visible packs submit flags as `(obj_flags & 0x0002) | token_hi` where `token_hi = source_word & 0xFF00`, so palette token payload is carried in bits `15:8`; both submitters apply CLUT override only when `(flags & ~0xF) != 0`, therefore world-lane bit `0x0002` alone never activates override. Live artifacts in this batch are conservative nearby helper renames (`0x8003a3b0 -> psx_world_draw_tint_fade_step`, `0x80038f10 -> psx_noop_frame_hook_38f10`, `0x80044018 -> psx_noop_frame_hook_44018`) plus targeted decompiler comments at `0x80041590`, `0x800415c0`, `0x80044e10`, `0x80044eb8`, `0x80031f34`, and `0x80031f3c`. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 focused live kind-5 image-table palette-path verification pass on active `SLUS_002.68` centered on `0x80044e9c` with world caller lanes at `0x800415c0` and `0x800412dc`, sibling sprite branch checks at `0x80044e10/0x80044e34/0x80044e5c`, and CLUT table uses at `0x800a9f48/0x800a9f66`. Current best read is now exporter-rule exact for this lane: image-table submit uses default `psx_clut_table_by_resource_bank[resource_bank]` when `(submit_flags & 0xfffffff0)==0`, otherwise uses `psx_clut_override_table_by_palette_token[(submit_flags>>8)]`; main-visible injects authored palette-token high byte while special-visible does not, so stage-2 world draw remains default-bank CLUT on this path. Live artifacts in this batch are one conservative nearby helper rename (`0x80044380 -> psx_clut_vram_rows_f0_f8_swap`) plus a decompiler comment preserving its global CLUT-page-swap role. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 focused live `SLUS_002.68` kind-4/kind-5 palette-selection field closure pass centered on `0x800444e4`, `0x80044614`, `0x80044bdc`, and `0x80044e9c` with lane-source checks at `0x80041458/0x80041144`. Current best read is now exporter-rule exact: default CLUT bank still comes from header `+0x14` through resource `+0x08`, no additional frame-table header field in bind/upload helpers selects palette directly, sprite override routing is format-dependent (`resource+0x04`), and high-byte palette token source remains draw-lane/object-authored rather than resource-header-local. Live artifacts in this batch are targeted decompiler comments at `0x800444e4`, `0x80044614`, `0x80044bdc`, and `0x80044e9c`. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 focused live `SLUS_002.68` sprite-submitter CLUT closure pass centered on `psx_sprite_resource_submit_frame` (`0x80044bdc`) with caller lanes `0x800415e0/0x800412f8` and bind/load feeders `0x800444e4/0x80044614`. Current best read is now rule-exact for exporter logic: sprite CLUT selection is format-branching (`resource+0x04==2` versus non-`2`) under a shared override gate (`submit_flags & 0xfffffff0`), main-visible injects authored high-byte palette token while special-visible does not, and the descriptor/header palette bank (`descriptor+0x14`) survives through `resource+0x08` to remain the default CLUT selector when no active override token is present. Live artifacts in this batch are one nearby helper rename (`0x80044380 -> psx_swap_clut_rows_f0_f8`) plus targeted decompiler comments at `0x80044380`, `0x800444fc`, `0x80044680`, `0x80044e1c`, `0x80044e34`, and `0x80044e4c`. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 focused live visibility-routing and main-visible ordering pass on active `SLUS_002.68` centered on `0x800131a8`, player/object neighbors in `0x8001263c..0x80013688`, and ordering helpers `0x8002be6c`, `0x8002c89c`, `0x8002ca74`, `0x8002d778`, and `0x8002e064` plus nearby graph-node helpers `0x8002cd60`, `0x8002ce38`, `0x8002cf3c`, and `0x8002cfb0`. Current best read is now exporter-actionable and lane/order split explicit: stage-1 vs stage-2 choice remains object-local (`type==4 || obj+0x1c bit0x0400`), policy-table reads are downstream ordering/publication controls rather than lane selectors, and stage-1 draw order is dependency-graph driven with policy-biased pair compare/unlink behavior rather than plain depth sort. Live artifacts in this batch are four conservative helper renames and seven targeted decompiler comments preserving route-split and graph-order semantics. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 live MCP world-visible draw lane and CLUT-routing refresh on active `SLUS_002.68` centered on `0x80041378`, `0x80041458`, `0x80041144`, `0x80044bdc`, `0x80044e9c`, and CLUT tables `0x800a9f48/0x800a9f66`. Current best read is route-explicit and exporter-critical: world draw order remains stage-1 main-visible then stage-2 special-visible then HUD; submitter dispatch remains resource-kind based for world lanes (`kind==5` image-table else sprite); main-visible injects authored palette-token high byte while special-visible does not; and CLUT override gating remains shared (`submit_flags & 0xfffffff0`) with lane-dependent table selection. Live artifacts in this batch are conservative helper renames `0x8002e534 -> psx_marker_channel_runtime_get_u16_86` and `0x8002eee8 -> psx_marker_channel_runtime_get_u16_84`, plus targeted decompiler comments at `0x80041378`, `0x800415c0`, `0x800412dc`, `0x80044ed0`, and `0x80044e5c`. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 focused live `SLUS_002.68` art-payload decode semantics pass centered on `psx_resource_bind_single_image_vram_slot` (`0x800444e4`), `image_bundle_load_to_vram` (`0x80044614`), `sprite_rle_decode_rows` (`0x80045264`), and frame-geometry helpers (`0x80045014`, `0x800450a8`, `0x8004513c`, `0x800451d0`). Current best read is standalone-exporter explicit: kind-4 is a single-slot descriptor bind lane, kind-5 is a per-frame table upload lane with raw-versus-RLE branch on frame flags bit0, row-RLE control semantics are now pinned for offline decode, and frame geometry/origin extraction must branch by resource kind and frame-table stride instead of visibility lane flags. Live artifacts in this batch are three conservative helper renames (`0x80045440`, `0x800455d4`, `0x80045d78`) plus targeted decompiler comments at the key kind-4/kind-5 bind, RLE decode, and frame-geometry entry points. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 live MCP authored-family descriptor/constructor closure pass on active `SLUS_002.68` centered on `0x800256b0`, `0x800258cc`, `0x800249f4`, `0x80024eec`, and descriptor row `0x800626f8`. Current best read is now convergence-explicit and exporter-facing: section-0 root and constructor-placement records for unresolved families (`0x0042`, `0x0049`, `0x0055..0x0063`) still converge through shared row `0x800626f8` slot0, while constructor divergence-relevant data remains authored route-word copy into `obj+0x1c` plus post-bind state/route/latch channels, not a type-unique descriptor callback split. Live artifacts in this batch are one conservative helper rename (`0x80031c34`) and targeted comments at `0x800256b0`, `0x800258cc`, `0x800249f4`, `0x80024eec`, and `0x800626f8`. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 live MCP selector-install/transition-reselection/final-latch closure pass on active `SLUS_002.68` centered on `0x800260e8`, `0x80025d68`, `0x8001bca0`, `0x80018578`, `0x8002906c`, and `0x80029dac`. Current best read is now exporter-critical and ordering-explicit: selector install (`obj+0x9e`) is pre-latch setup, final visible token (`obj+0x94`) is latched later in state advance, transition-table/type-`0x0042` paths mutate selector and narrow low control bits before that latch, and the unresolved post-construction path near `FUN_8002906c` is closed as delayed trigger into `psx_type4_reselect_motion_state`. Live artifacts in this batch are targeted decompiler comments at `0x800260e8`, `0x80025d68`, `0x8001bca0`, `0x80018578`, `0x8002906c`, and `0x80029dac`. + +Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-13 focused live `SLUS_002.68` loader/bundle-install and compressed-state lane pass centered on `wdl_resource_bundle_load_by_index` (`0x80039444`), `psx_stream_install_type_runtime_banks` (`0x80038f18`), `psx_install_type_state_script_component_extents_banks` (`0x8003917c`), `psx_install_type_art_active_header_and_built_resource` (`0x80045ffc`), `psx_install_level_audio_runtime_stream_bundle` (`0x80040768`), and `psx_lzss_unpack_into_level_buffer` (`0x8003b00c`). Current best read is exporter-explicit: stream-installer per-type records seed state/script/component/extents plus raw active-header pointer while clearing built-resource slots; WDL install path separately resolves built resources via kind-4/5 art install; detached runtime-stream blob carries a fixed header-driven multi-chunk install (9 lane arrays + SPU tail upload); and compressed-state inflate remains a pre-dispatch `0x3e00` lane with zero-token termination feeding runtime-header apply/root dispatch. Live artifacts in this batch are three conservative helper renames (`0x8002b6b8`, `0x8002b6e0`, `0x80024720`) plus targeted decompiler comments at `0x80038f18`, `0x80040768`, `0x8003b00c`, and `0x80024720`. + Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-12 live MCP CLUT override routing closure pass on active `SLUS_002.68` centered on `0x80041458`, `0x80041144`, `0x80044bdc`, `0x80044e9c`, `0x800a9f48`, and `0x800a9f66`. Current best read is now exporter-critical and path-explicit: main-visible injects authored palette token while special-visible does not; submitter override gate is shared (`flags & 0xfffffff0`); and active override resolution diverges by submitter/resource-format lane (image-table and sprite format-2 use `psx_clut_override_table_by_palette_token[token]`, sprite non-format-2 uses token as a row key into `psx_clut_table_by_resource_bank`). Practical consequence is that token `0` is effectively no-override for this world-object path and exporter CLUT logic must branch by route lane plus submitter/resource format instead of flattening token handling. Live artifacts in this batch are targeted comments at `0x800415b0`, `0x800412d0`, `0x80044e10`, and `0x80044eb8`. Latest verified batch: [docs/psx/art-binding-recovery.md](docs/psx/art-binding-recovery.md) now includes a 2026-04-12 live MCP wall-family discriminator pass for the exporter regression where atlases repeat and wall faces collapse. Current best read is now split-explicit for `0x003e..0x004f`: constructor bind in `psx_object_create_simple_record`/`psx_object_create_compound_record` still converges on per-type `DAT_800758d8[type]`, while real divergence happens post-bind through selector install (`0x800260e8`), frame-token latch (`0x80025d68` -> `obj+0x94`), and stage-1 versus stage-2 route semantics (`0x80041458` vs `0x80041144`), including main-visible-only authored palette-token injection for `>=0x003e`. Immediate exporter consequence is to prioritize effective route/latch-state discrimination over inventing a new pre-constructor resource-bank split. Live artifacts in this batch are targeted comments at `0x80046038`, `0x80026100`, `0x80041554`, and `0x80040f88`. diff --git a/psx-map-exporter/docs/spec.md b/psx-map-exporter/docs/spec.md index bff71cd..2dfef44 100644 --- a/psx-map-exporter/docs/spec.md +++ b/psx-map-exporter/docs/spec.md @@ -184,6 +184,14 @@ For the generic family band now dominating `LSET1/L0` failures (`0x003e`, `0x004 The chosen bundle and clamped frame index, plus binding-diversity metrics, are preserved in output metadata so failures stay auditable. +There is now one opt-in experimental binding mode for current map-0 research: + +- `runtime-map0-masked-proxy` + +That mode reads `.cache/runtime-map0-correlation.json`, takes the live `headerWord11` field from the current map-0 type rows, masks it to `0x0fffff`, and remaps a type only when that masked value lands within a small tolerance of a scanned raw bundle offset with matching kind/mode. All non-matching types still fall back to the raw slot rule. + +This is still a probe rule, not claimed final executable truth. It exists to turn the new RAM-backed map-0 correlation into a small, auditable extraction improvement without pretending the full late `DAT_800758d8` bank parse is solved. + When debug labels are enabled for a map render, labels now identify unique rendered resources rather than per-instance placements. The stable label key is currently `bundle offset + clamped frame + resolved palette`. Validation atlas sheets still use progressive cell indices. ## Rendering Rule @@ -231,6 +239,7 @@ Supported options: - `--source ` - `--wdl ` - `--disc-root ` +- `--binding-mode ` - `--map-source ` - `--out-name ` diff --git a/psx-map-exporter/src/cli.js b/psx-map-exporter/src/cli.js index 52e3e38..fb1f502 100644 --- a/psx-map-exporter/src/cli.js +++ b/psx-map-exporter/src/cli.js @@ -24,6 +24,7 @@ function parseArgs(argv) { 'Crusader - No Remorse (USA) GPU RAM 2.bin' ), mapSource: 'auto', + bindingMode: 'raw', sceneScope: 'probe', validationBundles: [], }; @@ -52,6 +53,11 @@ function parseArgs(argv) { index += 1; continue; } + if (arg === '--binding-mode') { + options.bindingMode = next; + index += 1; + continue; + } if (arg === '--scene-scope') { options.sceneScope = next; index += 1; @@ -97,6 +103,7 @@ function printHelp() { ' --source WDL path relative to the PSX disc root', ' --wdl Direct WDL path', ' --disc-root PSX asset root, defaults to STATIC_PSX in the sibling workspace', + ' --binding-mode Raw slot binding by default; optional map-0 runtime proxy uses .cache/runtime-map0-correlation.json when present', ' --scene-scope Probe is supported; full is intentionally disabled until raw floor and full-map decode is recovered', ' --gpu-ram-dump PSX GPU RAM dump used for live mode-1 palette extraction', ' --validation-bundles Comma-separated bundle absolute offsets (hex or decimal) for bundle palette-sweep validation sheets', @@ -133,6 +140,7 @@ async function main() { wdlPath, sourceRelPath: options.source, mapSource: options.mapSource, + bindingMode: options.bindingMode, sceneScope: options.sceneScope, gpuRamDumpPath: options.gpuRamDump, validationBundles: options.validationBundles, diff --git a/psx-map-exporter/src/export-map.js b/psx-map-exporter/src/export-map.js index 754408d..9497bc1 100644 --- a/psx-map-exporter/src/export-map.js +++ b/psx-map-exporter/src/export-map.js @@ -119,6 +119,132 @@ function chooseBundleForType(bundles, typeWord) { return null; } +async function loadJsonIfExists(filePath) { + try { + return JSON.parse(await fs.readFile(filePath, 'utf8')); + } catch { + return null; + } +} + +function buildRuntimeMap0MaskedBindings(correlation, bundles, tolerance = 0x200) { + const byType = new Map(); + const diagnostics = []; + const provisionalBindings = []; + + if (!correlation || correlation.currentMapId !== 0 || !Array.isArray(correlation.combined)) { + return { byType, diagnostics }; + } + + for (const row of correlation.combined) { + if (!row || !Number.isInteger(row.typeId) || !Number.isInteger(row.rawRecordCount) || row.rawRecordCount <= 0) { + continue; + } + if ((row.headerKind !== 4 && row.headerKind !== 5) || (row.headerWord3 !== 1 && row.headerWord3 !== 2)) { + continue; + } + if (!Number.isInteger(row.headerWord11) || row.headerWord11 === 0) { + continue; + } + + const maskedAbsoluteOffset = row.headerWord11 & 0x0fffff; + let bestMatch = null; + for (const bundle of bundles) { + if (bundle.kind !== row.headerKind || bundle.mode !== row.headerWord3) { + continue; + } + + const absoluteOffsetDelta = Math.abs(bundle.absoluteOffset - maskedAbsoluteOffset); + if (!bestMatch || absoluteOffsetDelta < bestMatch.absoluteOffsetDelta) { + bestMatch = { + bundle, + absoluteOffsetDelta, + }; + } + } + + if (!bestMatch || bestMatch.absoluteOffsetDelta > tolerance) { + continue; + } + + const binding = { + typeId: row.typeId, + bundle: bestMatch.bundle, + mappingSource: 'runtime-map0-masked-header-offset-proxy', + runtimeBinding: { + maskedAbsoluteOffset, + absoluteOffsetDelta: bestMatch.absoluteOffsetDelta, + headerKind: row.headerKind, + headerMode: row.headerWord3, + headerWord4: row.headerWord4, + headerWord8: row.headerWord8, + headerWord10: row.headerWord10, + visibleCount: row.visibleCount ?? 0, + rawRecordCount: row.rawRecordCount, + }, + }; + + provisionalBindings.push(binding); + } + + const bundleBuckets = new Map(); + for (const binding of provisionalBindings) { + const bucket = bundleBuckets.get(binding.bundle.absoluteOffset) ?? []; + bucket.push(binding); + bundleBuckets.set(binding.bundle.absoluteOffset, bucket); + } + + for (const binding of provisionalBindings) { + const bucket = bundleBuckets.get(binding.bundle.absoluteOffset) ?? []; + const isCrowdedLargeSingleFrameBundle = + bucket.length >= 4 && + binding.bundle.frameCount === 1 && + binding.bundle.width >= 96 && + binding.bundle.height >= 48; + + if (isCrowdedLargeSingleFrameBundle) { + continue; + } + + byType.set(binding.typeId, binding); + diagnostics.push({ + typeId: binding.typeId, + bundleSlot: binding.bundle.slot, + bundleAbsoluteOffset: binding.bundle.absoluteOffset, + maskedAbsoluteOffset: binding.runtimeBinding.maskedAbsoluteOffset, + absoluteOffsetDelta: binding.runtimeBinding.absoluteOffsetDelta, + headerKind: binding.runtimeBinding.headerKind, + headerMode: binding.runtimeBinding.headerMode, + visibleCount: binding.runtimeBinding.visibleCount, + rawRecordCount: binding.runtimeBinding.rawRecordCount, + crowdedBundleTypeCount: bucket.length, + }); + } + + diagnostics.sort((left, right) => left.typeId - right.typeId); + return { byType, diagnostics }; +} + +function chooseBundleBinding(record, bundles, options = {}) { + if (options.bindingMode === 'runtime-map0-masked-proxy') { + const runtimeBinding = options.runtimeMap0Bindings?.get(record.typeWord) ?? null; + if (runtimeBinding?.bundle) { + return runtimeBinding; + } + } + + const rawTypeBundle = chooseBundleForType(bundles, record.typeWord); + if (!rawTypeBundle) { + return null; + } + + return { + bundle: rawTypeBundle, + mappingSource: 'raw-typeword-bundle-slot-diagnostic', + runtimeBinding: null, + }; +} + function describeMapScope(recordSet) { if (recordSet.source === 'combined') { return 'layered object-projection probe from both loader-sized section-0 constructor-placement and root-dispatch records in post_audio_section_00'; @@ -738,9 +864,9 @@ async function buildSceneItems(region04, records, bundles, options = {}) { continue; } - const rawTypeBundle = chooseBundleForType(bundles, record.typeWord); - const bundle = rawTypeBundle; - if (!bundle) { + const binding = chooseBundleBinding(record, bundles, options); + const bundle = binding?.bundle ?? null; + if (!bundle || !binding) { continue; } if (nonMapFacingBundleOffsets.has(bundle.absoluteOffset)) { @@ -788,9 +914,13 @@ async function buildSceneItems(region04, records, bundles, options = {}) { defaultPaletteIndex: bundle.defaultPaletteIndex ?? null, resolvedPaletteIndex: bundle.resolvedPaletteIndex ?? null, paletteFormula: bundle.paletteFormula ?? null, - mappingSource: 'raw-typeword-bundle-slot-diagnostic', + mappingSource: binding.mappingSource, templateTypeId: null, donorTypeId: null, + runtimeBindingMaskedAbsoluteOffset: binding.runtimeBinding?.maskedAbsoluteOffset ?? null, + runtimeBindingOffsetDelta: binding.runtimeBinding?.absoluteOffsetDelta ?? null, + runtimeBindingVisibleCount: binding.runtimeBinding?.visibleCount ?? null, + runtimeBindingRawRecordCount: binding.runtimeBinding?.rawRecordCount ?? null, rawWords: record.rawWords ?? record.words, flipped: (record.laneWord & 0x0002) !== 0, width: sprite.width, @@ -903,10 +1033,17 @@ export async function exportMap(options) { { mode1RuntimePalette } ); + const runtimeMap0Correlation = options.bindingMode === 'runtime-map0-masked-proxy' + ? await loadJsonIfExists(path.join(cacheBaseRoot, 'runtime-map0-correlation.json')) + : null; + const runtimeMap0BindingResult = buildRuntimeMap0MaskedBindings(runtimeMap0Correlation, bundles); + const { items: sceneItems, skippedRecords } = await buildSceneItems(region04, recordSet.records, bundles, { paletteSets, + bindingMode: options.bindingMode, mode1RuntimePalette, mode1PaletteBank, + runtimeMap0Bindings: runtimeMap0BindingResult.byType, }); const bindingDiversity = summarizeBindingDiversity(sceneItems); const authoredLayerSummary = summarizeAuthoredLayers(recordSet.records); @@ -949,7 +1086,11 @@ export async function exportMap(options) { bundleCount: bundles.length, bundleSource: bundles[0]?.bundleSource ?? 'none', gpuRamDumpPath: options.gpuRamDumpPath ?? null, - artBindingSource: 'raw-typeword-bundle-slot-diagnostic', + artBindingSource: options.bindingMode === 'runtime-map0-masked-proxy' + ? 'runtime-map0-masked-header-offset-proxy-with-raw-fallback' + : 'raw-typeword-bundle-slot-diagnostic', + runtimeMap0BindingTypeCount: runtimeMap0BindingResult.diagnostics.length, + runtimeMap0BindingTypes: runtimeMap0BindingResult.diagnostics, activeHeaderOverrideCandidateCount: activeHeaderOverrideCandidates.length, bestActiveHeaderOverrideCandidate: activeHeaderOverrideCandidates[0] ? { @@ -975,7 +1116,9 @@ export async function exportMap(options) { 'The root-dispatch lane is now rendered as a second authored layer, but runtime-driven control mutations and dynamic effect spawns are still out of scope.', 'Known non-map-facing portrait/talk root types `0x0042` and `0x0049`, plus the known portrait bundle `0x000D84F4`, are excluded from probe rendering.', 'Viewer-derived sidecars, donor mappings, and cached scene references are intentionally disabled in this standalone exporter.', - 'Current bundle selection is still diagnostic-only until the late DAT_800758d8 art bank is parsed directly.', + options.bindingMode === 'runtime-map0-masked-proxy' + ? 'Current bundle selection uses an experimental map-0 runtime masked-offset proxy where live headerWord11 low bits land near a scanned raw bundle; all non-matching types still fall back to the raw slot diagnostic.' + : 'Current bundle selection is still diagnostic-only until the late DAT_800758d8 art bank is parsed directly.', 'No floor or tile layer is decoded directly yet; post_audio_region_02 and the decompressed level-state lane remain unresolved.', ]), 'Palette routing remains partly heuristic when authored token and default bank evidence are both absent.', @@ -1023,6 +1166,10 @@ export async function exportMap(options) { mappingSource: item.mappingSource, templateTypeId: item.templateTypeId, donorTypeId: item.donorTypeId, + runtimeBindingMaskedAbsoluteOffset: item.runtimeBindingMaskedAbsoluteOffset, + runtimeBindingOffsetDelta: item.runtimeBindingOffsetDelta, + runtimeBindingVisibleCount: item.runtimeBindingVisibleCount, + runtimeBindingRawRecordCount: item.runtimeBindingRawRecordCount, token06HighByte: item.paletteDiagnostics?.token06HighByte ?? null, token0cHighByte: item.paletteDiagnostics?.token0cHighByte ?? null, expectedPaletteToken: item.paletteDiagnostics?.expectedPaletteToken ?? null, diff --git a/psx-map-exporter/tmp_correlate_runtime_map0.mjs b/psx-map-exporter/tmp_correlate_runtime_map0.mjs new file mode 100644 index 0000000..8ec6234 --- /dev/null +++ b/psx-map-exporter/tmp_correlate_runtime_map0.mjs @@ -0,0 +1,79 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +const projectRoot = path.resolve('..'); +const runtimePath = path.resolve(projectRoot, 'psx-map-exporter', '.cache', 'runtime-snapshot-report.json'); +const recordsPath = path.resolve(projectRoot, 'psx-map-exporter', '.cache', 'ctor_stream_probe', 'records.json'); + +const runtime = JSON.parse(fs.readFileSync(runtimePath, 'utf8')); +const records = JSON.parse(fs.readFileSync(recordsPath, 'utf8')); + +const visibleObjects = runtime.sampleVisibleObjects ?? []; +const visibleTypeMap = new Map(); +for (const object of visibleObjects) { + const key = object.typeId; + const entry = visibleTypeMap.get(key) ?? { + typeId: key, + count: 0, + routeWords: new Set(), + selectorIndexes: new Set(), + latchedTokens: new Set(), + artResourcePtrs: new Set(), + authoredRecordPtrs: [], + }; + entry.count += 1; + entry.routeWords.add(object.routeWord); + entry.selectorIndexes.add(object.selectorIndex); + entry.latchedTokens.add(object.latchedToken); + entry.artResourcePtrs.add(object.artResourcePtr); + if (entry.authoredRecordPtrs.length < 8) { + entry.authoredRecordPtrs.push(object.authoredRecordPtr); + } + visibleTypeMap.set(key, entry); +} + +const typeRows = new Map((runtime.sampleTypeRows ?? []).map((row) => [row.typeId, row])); +const recordTypeCounts = new Map(); +const recordLaneByType = new Map(); +for (const record of records.records ?? []) { + recordTypeCounts.set(record.typeWord, (recordTypeCounts.get(record.typeWord) ?? 0) + 1); + const laneSet = recordLaneByType.get(record.typeWord) ?? new Set(); + laneSet.add(record.laneWord); + recordLaneByType.set(record.typeWord, laneSet); +} + +const combined = [...new Set([...recordTypeCounts.keys(), ...visibleTypeMap.keys()])] + .sort((left, right) => left - right) + .map((typeId) => { + const visible = visibleTypeMap.get(typeId); + const row = typeRows.get(typeId) ?? null; + return { + typeId, + rawRecordCount: recordTypeCounts.get(typeId) ?? 0, + rawLaneWords: [...(recordLaneByType.get(typeId) ?? new Set())].sort((a, b) => a - b), + visibleCount: visible?.count ?? 0, + visibleRouteWords: visible ? [...visible.routeWords].sort((a, b) => a - b) : [], + visibleSelectorIndexes: visible ? [...visible.selectorIndexes].sort((a, b) => a - b) : [], + visibleLatchedTokens: visible ? [...visible.latchedTokens].sort((a, b) => a - b) : [], + visibleArtResourcePtrs: visible ? [...visible.artResourcePtrs] : [], + sampleAuthoredRecordPtrs: visible?.authoredRecordPtrs ?? [], + activeHeader: row?.activeHeader ?? 0, + builtResource: row?.builtResource ?? 0, + headerKind: row?.headerWords?.[0] ?? null, + headerWord3: row?.headerWords?.[3] ?? null, + headerWord4: row?.headerWords?.[4] ?? null, + headerWord8: row?.headerWords?.[8] ?? null, + headerWord10: row?.headerWords?.[10] ?? null, + headerWord11: row?.headerWords?.[11] ?? null, + }; + }) + .filter((entry) => entry.rawRecordCount !== 0 || entry.visibleCount !== 0); + +const outputPath = path.resolve(projectRoot, 'psx-map-exporter', '.cache', 'runtime-map0-correlation.json'); +fs.writeFileSync(outputPath, JSON.stringify({ + currentMapId: runtime.currentMapId, + visibleObjectCount: runtime.visibleObjectCount, + combined, +}, null, 2)); + +console.log(JSON.stringify({ outputPath }, null, 2)); \ No newline at end of file diff --git a/psx-map-exporter/tmp_dump_runtime_snapshot.mjs b/psx-map-exporter/tmp_dump_runtime_snapshot.mjs new file mode 100644 index 0000000..4841bdd --- /dev/null +++ b/psx-map-exporter/tmp_dump_runtime_snapshot.mjs @@ -0,0 +1,159 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +const RAM_BASE = 0x80000000; +const TYPE_TABLE_BASE = 0x800758c4; +const TYPE_ROW_STRIDE = 0x18; +const MAIN_VISIBLE_LIST = 0x8006ad5c; +const MAIN_VISIBLE_READ_INDEX = 0x80067690; +const MAIN_VISIBLE_WRITE_INDEX = 0x800676bc; +const CURRENT_MAP_ID = 0x80067728; +const NEXT_MAP_ID = 0x800678d0; + +const projectRoot = path.resolve('..'); +const ramDumpPath = path.resolve(projectRoot, 'binary', 'Crusader - No Remorse (USA) Main Memory Dump.bin'); +const ram = fs.readFileSync(ramDumpPath); +const outputPath = path.resolve(projectRoot, 'psx-map-exporter', '.cache', 'runtime-snapshot-report.json'); + +function vaToOffset(address) { + const offset = address - RAM_BASE; + if (offset < 0 || offset >= ram.length) { + throw new RangeError(`Address out of dump range: 0x${address.toString(16)}`); + } + return offset; +} + +function canRead(address, size = 4) { + const offset = address - RAM_BASE; + return offset >= 0 && offset + size <= ram.length; +} + +function readU32(address) { + return ram.readUInt32LE(vaToOffset(address)); +} + +function readS32(address) { + return ram.readInt32LE(vaToOffset(address)); +} + +function readU16(address) { + return ram.readUInt16LE(vaToOffset(address)); +} + +function readS16(address) { + return ram.readInt16LE(vaToOffset(address)); +} + +function readHeaderWords(address, wordCount = 10) { + if (!canRead(address, wordCount * 4)) { + return null; + } + return Array.from({ length: wordCount }, (_, index) => readU32(address + index * 4)); +} + +function parseTypeRows(limit = 0x100) { + const rows = []; + for (let typeId = 0; typeId < limit; typeId += 1) { + const rowBase = TYPE_TABLE_BASE + typeId * TYPE_ROW_STRIDE; + const installCount = readU32(rowBase + 0x00); + const builtResource = readU32(rowBase + 0x04); + const stateScript = readU32(rowBase + 0x08); + const simpleComponent = readU32(rowBase + 0x0c); + const extents = readU32(rowBase + 0x10); + const activeHeader = readU32(rowBase + 0x14); + + if ((installCount | builtResource | stateScript | simpleComponent | extents | activeHeader) === 0) { + continue; + } + + const headerWords = activeHeader !== 0 ? readHeaderWords(activeHeader, 12) : null; + rows.push({ + typeId, + rowBase, + installCount, + builtResource, + stateScript, + simpleComponent, + extents, + activeHeader, + headerWords, + }); + } + return rows; +} + +function parseVisibleObjects(limit = 256) { + const readIndex = readU32(MAIN_VISIBLE_READ_INDEX); + const writeIndex = readU32(MAIN_VISIBLE_WRITE_INDEX); + const count = Math.max(0, Math.min(limit, writeIndex)); + const objects = []; + + for (let index = 0; index < count; index += 1) { + const objectPtr = readU32(MAIN_VISIBLE_LIST + index * 4); + if (!canRead(objectPtr, 0xa4)) { + continue; + } + + objects.push({ + index, + objectPtr, + typeId: readU16(objectPtr + 0x18), + routeWord: readU16(objectPtr + 0x1c), + stateFlags: readU16(objectPtr + 0x1e), + screenLeft: readS16(objectPtr + 0x20), + screenTop: readS16(objectPtr + 0x22), + screenRight: readS16(objectPtr + 0x24), + screenBottom: readS16(objectPtr + 0x26), + worldX: readS32(objectPtr + 0x3c), + worldY: readS32(objectPtr + 0x40), + worldZ: readS32(objectPtr + 0x44), + velocityX: readS32(objectPtr + 0x60), + velocityY: readS32(objectPtr + 0x64), + velocityZ: readS32(objectPtr + 0x68), + programPtr: readU32(objectPtr + 0x08), + artResourcePtr: readU32(objectPtr + 0x10), + companionExtentsPtr: readU32(objectPtr + 0x84), + stateScriptPtr: readU32(objectPtr + 0x88), + scriptBasePtr: readU32(objectPtr + 0x8c), + scriptReadPtr: readU32(objectPtr + 0x90), + latchedToken: readU16(objectPtr + 0x94), + scriptCountdown: readU16(objectPtr + 0x96), + selectorIndex: readU16(objectPtr + 0x9e), + authoredRecordPtr: readU32(objectPtr + 0xa0), + }); + } + + return { + readIndex, + writeIndex, + count, + objects, + }; +} + +const typeRows = parseTypeRows(); +const visible = parseVisibleObjects(); +const byType = new Map(); +for (const object of visible.objects) { + byType.set(object.typeId, (byType.get(object.typeId) ?? 0) + 1); +} + +const summary = { + ramDumpPath, + ramSize: ram.length, + currentMapId: readU32(CURRENT_MAP_ID), + nextMapId: readU32(NEXT_MAP_ID), + mainVisibleReadIndex: visible.readIndex, + mainVisibleWriteIndex: visible.writeIndex, + visibleObjectCount: visible.objects.length, + nonZeroTypeRowCount: typeRows.length, + visibleTypeCounts: [...byType.entries()] + .map(([typeId, count]) => ({ typeId, count })) + .sort((left, right) => right.count - left.count || left.typeId - right.typeId), + sampleTypeRows: typeRows.slice(0, 64), + sampleVisibleObjects: visible.objects.slice(0, 128), +}; + +fs.mkdirSync(path.dirname(outputPath), { recursive: true }); +fs.writeFileSync(outputPath, JSON.stringify(summary, null, 2)); +console.log(JSON.stringify({ outputPath }, null, 2)); \ No newline at end of file