Add new modules for Crusader map rendering and processing

- Implemented `formats.py` to define data structures and functions for handling map data, including reading and decoding shape and map items.
- Created `png.py` for generating PNG images from shape frames and pixel data.
- Developed `sorting.py` to manage the sorting and rendering order of map items based on their properties and spatial relationships.
- Introduced `render_all_maps.py` to facilitate the rendering of all maps for specified games, including command-line argument parsing and subprocess management for rendering tasks.

tools/crusader_map/__init__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+__all__ = ["main"]

tools/crusader_map/cli.py

@@ -0,0 +1,261 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+from .formats import (
+    FLAG_FLIPPED,
+    ShapeArchive,
+    collect_render_items,
+    load_globs,
+    load_map_items,
+    load_palette,
+    load_typeflags,
+    parse_world_rect,
+    resolve_fixed_dat,
+    resolve_static_dir,
+)
+from .png import DEFAULT_BACKGROUND, blit_frame, rgba_buffer, write_png_rgba
+from .sorting import prepare_sorted_items
+
+
+KNOWN_MAP_USAGE_HINTS = {
+    "remorse": {
+        0: [
+            "ScummVM CruGame::startGame() calls World::switchMap(0) for a new No Remorse game.",
+            "The same startup path comments the initial player placement as 'Map 1 (mission 1)', so this is a confirmed mission-start map anchor.",
+        ],
+    },
+    "regret": {},
+}
+
+
+def summarize_render_classes(base_items: list, shape_infos: list) -> dict[str, int]:
+    summary = {
+        "roof_items": 0,
+        "editor_items": 0,
+        "egg_family_items": 0,
+        "invisible_flagged_items": 0,
+        "npc_linked_items": 0,
+    }
+    for item in base_items:
+        if item.flags & 0x0010:
+            summary["invisible_flagged_items"] += 1
+        if item.npc_num != 0:
+            summary["npc_linked_items"] += 1
+        if item.shape >= len(shape_infos):
+            continue
+        info = shape_infos[item.shape]
+        if info.is_roof:
+            summary["roof_items"] += 1
+        if info.is_editor:
+            summary["editor_items"] += 1
+        if info.family in (3, 4, 7, 8):
+            summary["egg_family_items"] += 1
+    return summary
+
+
+def map_usage_info(game: str, map_index: int, base_items: list, render_items: list) -> dict[str, object]:
+    hints = KNOWN_MAP_USAGE_HINTS.get(game, {}).get(map_index, [])
+    item_map_nums = sorted({item.map_num for item in base_items})
+    nonzero_item_map_nums = [value for value in item_map_nums if value != 0]
+    npc_count = sum(1 for item in base_items if item.npc_num != 0)
+    return {
+        "status": "known_used" if hints else "unknown",
+        "confidence": "commented_reference" if hints else "unknown",
+        "known_hints": hints,
+        "item_map_nums": item_map_nums,
+        "nonzero_item_map_nums": nonzero_item_map_nums,
+        "npc_linked_item_count": npc_count,
+        "note": "No authoritative mission-to-map ownership table has been extracted yet. Unknown does not imply orphaned.",
+        "has_renderable_content": bool(render_items),
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Render a Crusader fixed map to PNG.")
+    parser.add_argument("--game", choices=("remorse", "regret"), default="remorse")
+    parser.add_argument("--static-dir", help="Override the static asset directory.")
+    parser.add_argument("--fixed-dat", help="Override the FIXED.DAT path when it does not live under the selected static directory.")
+    parser.add_argument("--map", dest="map_index", type=int, required=True, help="Map index to render.")
+    parser.add_argument("--output", required=True, help="PNG output path.")
+    parser.add_argument("--metadata", help="Optional JSON metadata output path.")
+    parser.add_argument("--no-globs", action="store_true", help="Disable GLOB.FLX expansion.")
+    parser.add_argument(
+        "--include-editor",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Render editor-only shapes. Enabled by default to keep debug/editor map content visible.",
+    )
+    parser.add_argument(
+        "--include-roofs",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        help="Render roof/exploration-obscurer shapes. Disabled by default.",
+    )
+    parser.add_argument(
+        "--include-hidden-markers",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Render hidden markers such as egg-family placements, editor/debug objects, and invisible marker shapes when they have visible frames.",
+    )
+    parser.add_argument(
+        "--world-rect",
+        nargs=4,
+        metavar=("MIN_X", "MIN_Y", "MAX_X", "MAX_Y"),
+        help="Restrict rendering to a world-space rectangle.",
+    )
+    parser.add_argument(
+        "--max-pixels",
+        type=int,
+        default=0,
+        help="Fail if the output image would exceed this many pixels. Non-positive values disable the limit.",
+    )
+    parser.add_argument(
+        "--progress-every",
+        type=int,
+        default=2000,
+        help="Emit collection and sorting progress every N items. Non-positive values disable progress logging.",
+    )
+    parser.add_argument(
+        "--invalid-detail-limit",
+        type=int,
+        default=20,
+        help="Maximum number of invalid shape/frame records to include in metadata.",
+    )
+    args = parser.parse_args()
+
+    repo_root = Path(__file__).resolve().parents[2]
+    static_dir = resolve_static_dir(repo_root, args.game, args.static_dir)
+    fixed_dat_path = resolve_fixed_dat(static_dir, args.fixed_dat)
+    world_rect = parse_world_rect(args.world_rect)
+
+    shape_infos = load_typeflags(static_dir / "TYPEFLAG.DAT")
+    palette = load_palette(static_dir / "GAMEPAL.PAL")
+    globs = load_globs(static_dir / "GLOB.FLX")
+    shape_archive = ShapeArchive(static_dir / "SHAPES.FLX")
+    progress_enabled = args.progress_every > 0
+    start_time = time.monotonic()
+
+    def log_progress(message: str) -> None:
+        if not progress_enabled:
+            return
+        elapsed = time.monotonic() - start_time
+        print(f"[map {args.map_index} +{elapsed:7.1f}s] {message}", file=sys.stderr, flush=True)
+
+    if not fixed_dat_path.exists():
+        raise FileNotFoundError(
+            f"missing FIXED.DAT at {fixed_dat_path}; copy the matching game map file into place or pass --fixed-dat"
+        )
+    base_items = load_map_items(fixed_dat_path, args.map_index)
+    log_progress(f"loaded {len(base_items)} fixed records from {fixed_dat_path}")
+    base_item_summary = summarize_render_classes(base_items, shape_infos)
+    render_items = collect_render_items(
+        base_items,
+        shape_infos,
+        globs,
+        include_editor=args.include_editor,
+        expand_globs=not args.no_globs,
+        world_rect=world_rect,
+        include_roofs=args.include_roofs,
+        include_hidden_markers=args.include_hidden_markers,
+        progress=log_progress if progress_enabled else None,
+        checkpoint_every=args.progress_every,
+    )
+    if not render_items:
+        raise ValueError("no renderable items were found for the selected map")
+
+    usage_info = map_usage_info(args.game, args.map_index, base_items, render_items)
+
+    min_left, min_top, max_right, max_bottom, prepared, occluded_count, invalid_item_count, invalid_items = prepare_sorted_items(
+        render_items,
+        shape_archive,
+        shape_infos,
+        progress=log_progress if progress_enabled else None,
+        checkpoint_every=args.progress_every,
+        max_invalid_details=args.invalid_detail_limit,
+    )
+    if not prepared:
+        raise ValueError("no valid shape/frame pairs were renderable for the selected map")
+    width = max_right - min_left
+    height = max_bottom - min_top
+    if width <= 0 or height <= 0:
+        raise ValueError("computed image bounds are invalid")
+    if args.max_pixels > 0 and width * height > args.max_pixels:
+        raise ValueError(
+            f"image would be {width}x{height} = {width * height} pixels; use --world-rect or raise --max-pixels"
+        )
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    buffer = rgba_buffer(width, height, DEFAULT_BACKGROUND)
+    for node_index, node in enumerate(prepared, start=1):
+        blit_frame(
+            buffer,
+            width,
+            height,
+            node.left - min_left,
+            node.top - min_top,
+            node.frame,
+            node.pixels,
+            palette,
+            flipped=bool(node.item.flags & FLAG_FLIPPED),
+        )
+        if progress_enabled and args.progress_every > 0 and node_index % args.progress_every == 0:
+            log_progress(f"blit painted={node_index} of {len(prepared)}")
+    write_png_rgba(output_path, width, height, buffer)
+    log_progress(f"wrote PNG {output_path} ({width}x{height})")
+
+    used_shapes = sorted({item.shape for item in render_items})
+    metadata = {
+        "game": args.game,
+        "static_dir": str(static_dir),
+        "fixed_dat": str(fixed_dat_path),
+        "map": args.map_index,
+        "raw_item_count": len(base_items),
+        "item_count": len(render_items),
+        "painted_item_count": len(prepared),
+        "occluded_item_count": occluded_count,
+        "invalid_item_count": invalid_item_count,
+        "invalid_items": [
+            {
+                "shape": item.shape,
+                "frame": item.frame,
+                "x": item.x,
+                "y": item.y,
+                "z": item.z,
+                "source": item.source,
+                "reason": item.reason,
+            }
+            for item in invalid_items
+        ],
+        "used_shape_count": len(used_shapes),
+        "used_shapes": used_shapes,
+        "usage": usage_info,
+        "base_item_summary": base_item_summary,
+        "sorter": "scummvm_dependency_graph",
+        "filters": {
+            "glob_expansion": not args.no_globs,
+            "editor_shapes_included": args.include_editor,
+            "roofs_included": args.include_roofs,
+            "hidden_markers_included": args.include_hidden_markers,
+        },
+        "bounds": {
+            "screen_left": min_left,
+            "screen_top": min_top,
+            "screen_right": max_right,
+            "screen_bottom": max_bottom,
+            "width": width,
+            "height": height,
+        },
+        "world_rect": list(world_rect) if world_rect else None,
+    }
+    if args.metadata:
+        metadata_path = Path(args.metadata)
+        metadata_path.parent.mkdir(parents=True, exist_ok=True)
+        metadata_path.write_text(json.dumps(metadata, indent=2), encoding="utf-8")
+    print(json.dumps(metadata, indent=2))
+    return 0

tools/crusader_map/formats.py

@@ -0,0 +1,516 @@
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable
+
+
+FLEX_TABLE_OFFSET = 0x80
+FLEX_COUNT_OFFSET = 0x54
+FIXED_MAP_COUNT_OFFSET = 0x54
+FIXED_MAP_TABLE_OFFSET = 0x80
+CRUSADER_COORD_SCALE = 2
+GLOB_COORD_MASK = ~0x3FF
+GLOB_COORD_SHIFT = 2
+GLOB_COORD_OFFSET = 2
+FLAG_INVISIBLE = 0x0010
+FLAG_FLIPPED = 0x0020
+EGG_FAMILIES = {3, 4, 7, 8}
+
+SI_FIXED = 0x0001
+SI_SOLID = 0x0002
+SI_LAND = 0x0008
+SI_OCCL = 0x0010
+SI_NOISY = 0x0080
+SI_DRAW = 0x0100
+SI_ROOF = 0x0400
+SI_TRANSL = 0x0800
+
+
+@dataclass(frozen=True)
+class FlexEntry:
+    offset: int
+    size: int
+
+
+@dataclass(frozen=True)
+class ShapeInfo:
+    family: int
+    flags: int
+    x: int
+    y: int
+    z: int
+    anim_type: int
+
+    @property
+    def is_editor(self) -> bool:
+        return bool(self.flags & 0x1000)
+
+    @property
+    def is_fixed(self) -> bool:
+        return bool(self.flags & SI_FIXED)
+
+    @property
+    def is_solid(self) -> bool:
+        return bool(self.flags & SI_SOLID)
+
+    @property
+    def is_land(self) -> bool:
+        return bool(self.flags & SI_LAND)
+
+    @property
+    def is_occl(self) -> bool:
+        return bool(self.flags & SI_OCCL)
+
+    @property
+    def is_noisy(self) -> bool:
+        return bool(self.flags & SI_NOISY)
+
+    @property
+    def is_draw(self) -> bool:
+        return bool(self.flags & SI_DRAW)
+
+    @property
+    def is_roof(self) -> bool:
+        return bool(self.flags & SI_ROOF)
+
+    @property
+    def is_translucent(self) -> bool:
+        return bool(self.flags & SI_TRANSL)
+
+    @property
+    def is_invitem(self) -> bool:
+        return self.family == 13
+
+
+@dataclass(frozen=True)
+class GlobItem:
+    x: int
+    y: int
+    z: int
+    shape: int
+    frame: int
+
+
+@dataclass(frozen=True)
+class MapItem:
+    x: int
+    y: int
+    z: int
+    shape: int
+    frame: int
+    flags: int
+    quality: int
+    npc_num: int
+    map_num: int
+    next_item: int
+    source: str
+
+
+@dataclass(frozen=True)
+class ShapeFrame:
+    compressed: bool
+    width: int
+    height: int
+    xoff: int
+    yoff: int
+    line_offsets: tuple[int, ...]
+    rle_data: bytes
+
+
+def read_u16_le(data: bytes, offset: int) -> int:
+    return struct.unpack_from("<H", data, offset)[0]
+
+
+def read_u24_le(data: bytes, offset: int) -> int:
+    return data[offset] | (data[offset + 1] << 8) | (data[offset + 2] << 16)
+
+
+def read_u32_le(data: bytes, offset: int) -> int:
+    return struct.unpack_from("<I", data, offset)[0]
+
+
+class FlexArchive:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.data = path.read_bytes()
+        self.entries = self._read_entries(self.data)
+
+    @staticmethod
+    def _read_entries(data: bytes) -> list[FlexEntry]:
+        count = read_u32_le(data, FLEX_COUNT_OFFSET)
+        entries: list[FlexEntry] = []
+        for index in range(count):
+            base = FLEX_TABLE_OFFSET + index * 8
+            entries.append(FlexEntry(read_u32_le(data, base), read_u32_le(data, base + 4)))
+        return entries
+
+    def get(self, index: int) -> bytes:
+        entry = self.entries[index]
+        if entry.size == 0:
+            return b""
+        return self.data[entry.offset : entry.offset + entry.size]
+
+    def __len__(self) -> int:
+        return len(self.entries)
+
+
+class ShapeArchive:
+    def __init__(self, path: Path) -> None:
+        self.archive = FlexArchive(path)
+        self._shape_cache: dict[int, tuple[ShapeFrame, ...]] = {}
+        self._decoded_frame_cache: dict[tuple[int, int], list[int]] = {}
+
+    def get_frame(self, shape_index: int, frame_index: int) -> ShapeFrame:
+        frames = self._get_shape(shape_index)
+        if frame_index < 0 or frame_index >= len(frames):
+            raise IndexError(f"shape {shape_index} frame {frame_index} out of range")
+        return frames[frame_index]
+
+    def decode_frame(self, shape_index: int, frame_index: int) -> tuple[ShapeFrame, list[int]]:
+        cache_key = (shape_index, frame_index)
+        decoded = self._decoded_frame_cache.get(cache_key)
+        frame = self.get_frame(shape_index, frame_index)
+        if decoded is None:
+            decoded = self._decode_pixels(frame)
+            self._decoded_frame_cache[cache_key] = decoded
+        return frame, decoded
+
+    def _get_shape(self, shape_index: int) -> tuple[ShapeFrame, ...]:
+        cached = self._shape_cache.get(shape_index)
+        if cached is not None:
+            return cached
+        raw = self.archive.get(shape_index)
+        if not raw:
+            raise ValueError(f"shape {shape_index} has no data")
+        frames = self._parse_shape(raw)
+        self._shape_cache[shape_index] = frames
+        return frames
+
+    @staticmethod
+    def _parse_shape(data: bytes) -> tuple[ShapeFrame, ...]:
+        frame_count = read_u16_le(data, 4)
+        frames: list[ShapeFrame] = []
+        for index in range(frame_count):
+            header_offset = 6 + index * 8
+            frame_offset = read_u24_le(data, header_offset)
+            frame_size = read_u32_le(data, header_offset + 4)
+            frame_data = data[frame_offset : frame_offset + frame_size]
+            if len(frame_data) < 28:
+                raise ValueError(f"frame {index} too small: {len(frame_data)}")
+            compressed = bool(read_u32_le(frame_data, 8))
+            width = read_u32_le(frame_data, 12)
+            height = read_u32_le(frame_data, 16)
+            xoff = struct.unpack_from("<i", frame_data, 20)[0]
+            yoff = struct.unpack_from("<i", frame_data, 24)[0]
+            line_offsets = tuple(read_u32_le(frame_data, 28 + row * 4) - ((height - row) * 4) for row in range(height))
+            rle_offset = 28 + height * 4
+            frames.append(
+                ShapeFrame(
+                    compressed=compressed,
+                    width=width,
+                    height=height,
+                    xoff=xoff,
+                    yoff=yoff,
+                    line_offsets=line_offsets,
+                    rle_data=frame_data[rle_offset:],
+                )
+            )
+        return tuple(frames)
+
+    @staticmethod
+    def _decode_pixels(frame: ShapeFrame) -> list[int]:
+        pixels = [-1] * (frame.width * frame.height)
+        rle = frame.rle_data
+        for row in range(frame.height):
+            pos = frame.line_offsets[row]
+            xpos = 0
+            while xpos < frame.width:
+                if pos >= len(rle):
+                    raise ValueError(f"row {row} overran RLE data")
+                xpos += rle[pos]
+                pos += 1
+                if xpos == frame.width:
+                    break
+                if pos >= len(rle):
+                    raise ValueError(f"row {row} missing run header")
+                dlen = rle[pos]
+                pos += 1
+                run_type = 0
+                if frame.compressed:
+                    run_type = dlen & 1
+                    dlen >>= 1
+                if dlen <= 0 or xpos + dlen > frame.width:
+                    raise ValueError(f"invalid run length {dlen} at row {row}")
+                row_base = row * frame.width + xpos
+                if run_type == 0:
+                    end = pos + dlen
+                    if end > len(rle):
+                        raise ValueError(f"row {row} literal run overruns RLE data")
+                    run = rle[pos:end]
+                    for index, color in enumerate(run):
+                        pixels[row_base + index] = color
+                    pos = end
+                else:
+                    if pos >= len(rle):
+                        raise ValueError(f"row {row} repeated-color run missing color byte")
+                    color = rle[pos]
+                    pos += 1
+                    for index in range(dlen):
+                        pixels[row_base + index] = color
+                xpos += dlen
+        return pixels
+
+
+def load_palette(path: Path) -> list[tuple[int, int, int]]:
+    data = path.read_bytes()
+    if len(data) < 768:
+        raise ValueError(f"palette too small: {path}")
+    palette: list[tuple[int, int, int]] = []
+    for index in range(256):
+        r = (data[index * 3] * 255) // 63
+        g = (data[index * 3 + 1] * 255) // 63
+        b = (data[index * 3 + 2] * 255) // 63
+        palette.append((r, g, b))
+    return palette
+
+
+def load_typeflags(path: Path) -> list[ShapeInfo]:
+    data = path.read_bytes()
+    infos: list[ShapeInfo] = []
+    for base in range(0, len(data), 9):
+        block = data[base : base + 9]
+        if len(block) < 9:
+            break
+        flags = 0
+        if block[0] & 0x01:
+            flags |= 0x0001
+        if block[0] & 0x02:
+            flags |= 0x0002
+        if block[0] & 0x04:
+            flags |= 0x0004
+        if block[0] & 0x08:
+            flags |= 0x0008
+        if block[0] & 0x10:
+            flags |= 0x0010
+        if block[0] & 0x20:
+            flags |= 0x0020
+        if block[0] & 0x40:
+            flags |= 0x0040
+        if block[0] & 0x80:
+            flags |= 0x0080
+        if block[1] & 0x01:
+            flags |= 0x0100
+        if block[1] & 0x02:
+            flags |= 0x0200
+        if block[1] & 0x04:
+            flags |= 0x0400
+        if block[1] & 0x08:
+            flags |= 0x0800
+        if block[6] & 0x01:
+            flags |= 0x1000
+        if block[6] & 0x02:
+            flags |= 0x2000
+        if block[6] & 0x04:
+            flags |= 0x4000
+        if block[6] & 0x08:
+            flags |= 0x8000
+        if block[6] & 0x10:
+            flags |= 0x10000
+        if block[6] & 0x20:
+            flags |= 0x20000
+        if block[6] & 0x40:
+            flags |= 0x40000
+        if block[6] & 0x80:
+            flags |= 0x80000
+        family = (block[1] >> 4) + ((block[2] & 1) << 4)
+        x = ((block[3] << 3) | (block[2] >> 5)) & 0x1F
+        y = (block[3] >> 2) & 0x1F
+        z = ((block[4] << 1) | (block[3] >> 7)) & 0x1F
+        anim_type = block[4] >> 4
+        infos.append(ShapeInfo(family=family, flags=flags, x=x, y=y, z=z, anim_type=anim_type))
+    return infos
+
+
+def load_globs(path: Path) -> list[list[GlobItem]]:
+    archive = FlexArchive(path)
+    globs: list[list[GlobItem]] = []
+    for index in range(len(archive)):
+        raw = archive.get(index)
+        if not raw:
+            globs.append([])
+            continue
+        count = read_u16_le(raw, 0)
+        items: list[GlobItem] = []
+        for item_index in range(count):
+            base = 2 + item_index * 6
+            items.append(
+                GlobItem(
+                    x=raw[base],
+                    y=raw[base + 1],
+                    z=raw[base + 2],
+                    shape=read_u16_le(raw, base + 3),
+                    frame=raw[base + 5],
+                )
+            )
+        globs.append(items)
+    return globs
+
+
+def load_map_items(path: Path, map_index: int) -> list[MapItem]:
+    if not path.exists():
+        raise FileNotFoundError(path)
+    data = path.read_bytes()
+    map_count = read_u16_le(data, FIXED_MAP_COUNT_OFFSET)
+    if map_index < 0 or map_index >= map_count:
+        raise ValueError(f"map index {map_index} out of range 0..{map_count - 1}")
+    table_offset = FIXED_MAP_TABLE_OFFSET + map_index * 8
+    map_offset = read_u32_le(data, table_offset)
+    map_size = read_u32_le(data, table_offset + 4)
+    payload = data[map_offset : map_offset + map_size]
+    if len(payload) != map_size:
+        raise ValueError(f"map {map_index} payload truncated")
+    items: list[MapItem] = []
+    for base in range(0, len(payload), 16):
+        record = payload[base : base + 16]
+        if len(record) < 16:
+            break
+        x = read_u16_le(record, 0) * CRUSADER_COORD_SCALE
+        y = read_u16_le(record, 2) * CRUSADER_COORD_SCALE
+        items.append(
+            MapItem(
+                x=x,
+                y=y,
+                z=record[4],
+                shape=read_u16_le(record, 5),
+                frame=record[7],
+                flags=read_u16_le(record, 8),
+                quality=read_u16_le(record, 10),
+                npc_num=record[12],
+                map_num=record[13],
+                next_item=read_u16_le(record, 14),
+                source="fixed",
+            )
+        )
+    return items
+
+
+def expand_glob_item(item: MapItem, globs: list[list[GlobItem]]) -> list[MapItem]:
+    if item.quality < 0 or item.quality >= len(globs):
+        return []
+    expanded: list[MapItem] = []
+    for glob_item in globs[item.quality]:
+        expanded.append(
+            MapItem(
+                x=(item.x & GLOB_COORD_MASK) + (glob_item.x << GLOB_COORD_SHIFT) + GLOB_COORD_OFFSET,
+                y=(item.y & GLOB_COORD_MASK) + (glob_item.y << GLOB_COORD_SHIFT) + GLOB_COORD_OFFSET,
+                z=item.z + glob_item.z,
+                shape=glob_item.shape,
+                frame=glob_item.frame,
+                flags=0,
+                quality=0,
+                npc_num=0,
+                map_num=item.map_num,
+                next_item=0,
+                source="glob",
+            )
+        )
+    return expanded
+
+
+def collect_render_items(
+    base_items: list[MapItem],
+    shape_infos: list[ShapeInfo],
+    globs: list[list[GlobItem]],
+    include_editor: bool,
+    expand_globs: bool,
+    world_rect: tuple[int, int, int, int] | None,
+    include_roofs: bool = True,
+    include_hidden_markers: bool = False,
+    progress: Callable[[str], None] | None = None,
+    checkpoint_every: int = 0,
+) -> list[MapItem]:
+    render_items: list[MapItem] = []
+    pending = list(base_items)
+    index = 0
+    skipped_invisible = 0
+    skipped_world_rect = 0
+    skipped_invalid_shape = 0
+    skipped_editor = 0
+    skipped_egg = 0
+    skipped_roof = 0
+    skipped_hidden = 0
+    expanded_globs = 0
+    while index < len(pending):
+        item = pending[index]
+        index += 1
+        if item.flags & FLAG_INVISIBLE:
+            if not include_hidden_markers:
+                skipped_hidden += 1
+                continue
+            skipped_invisible += 1
+        if world_rect is not None:
+            min_x, min_y, max_x, max_y = world_rect
+            if item.x < min_x or item.y < min_y or item.x > max_x or item.y > max_y:
+                skipped_world_rect += 1
+                continue
+        if item.shape >= len(shape_infos):
+            skipped_invalid_shape += 1
+            continue
+        info = shape_infos[item.shape]
+        if info.is_editor and not include_editor:
+            skipped_editor += 1
+            continue
+        if info.is_roof and not include_roofs:
+            skipped_roof += 1
+            continue
+        if expand_globs and info.family == 3 and item.source == "fixed":
+            pending.extend(expand_glob_item(item, globs))
+            expanded_globs += 1
+            if not include_hidden_markers:
+                continue
+        if info.family in EGG_FAMILIES and not include_hidden_markers:
+            skipped_egg += 1
+            continue
+        render_items.append(item)
+        if progress is not None and checkpoint_every > 0 and index % checkpoint_every == 0:
+            progress(
+                "collect "
+                f"processed={index} pending={len(pending)} rendered={len(render_items)} "
+                f"expanded_globs={expanded_globs} skipped=(invisible:{skipped_invisible}, world:{skipped_world_rect}, "
+                f"shape:{skipped_invalid_shape}, editor:{skipped_editor}, roof:{skipped_roof}, hidden:{skipped_hidden}, egg:{skipped_egg})"
+            )
+    if progress is not None:
+        progress(
+            "collect complete "
+            f"processed={index} pending={len(pending)} rendered={len(render_items)} "
+            f"expanded_globs={expanded_globs} skipped=(invisible:{skipped_invisible}, world:{skipped_world_rect}, "
+            f"shape:{skipped_invalid_shape}, editor:{skipped_editor}, roof:{skipped_roof}, hidden:{skipped_hidden}, egg:{skipped_egg})"
+        )
+    return render_items
+
+
+def parse_world_rect(values: list[str] | None) -> tuple[int, int, int, int] | None:
+    if values is None:
+        return None
+    if len(values) != 4:
+        raise ValueError("--world-rect expects four integers: min_x min_y max_x max_y")
+    min_x, min_y, max_x, max_y = (int(value, 0) for value in values)
+    if min_x > max_x or min_y > max_y:
+        raise ValueError("invalid --world-rect bounds")
+    return min_x, min_y, max_x, max_y
+
+
+def resolve_fixed_dat(static_dir: Path, fixed_dat: str | None) -> Path:
+    if fixed_dat:
+        return Path(fixed_dat)
+    return static_dir / "FIXED.DAT"
+
+
+def resolve_static_dir(repo_root: Path, game: str, static_dir: str | None) -> Path:
+    if static_dir:
+        return Path(static_dir)
+    if game == "regret":
+        return repo_root / "STATIC_REGRET"
+    return repo_root / "STATIC"

tools/crusader_map/png.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import struct
+import zlib
+from pathlib import Path
+
+from .formats import ShapeFrame
+
+
+DEFAULT_BACKGROUND = (10, 12, 18, 255)
+
+
+def rgba_buffer(width: int, height: int, color: tuple[int, int, int, int]) -> bytearray:
+    r, g, b, a = color
+    row = bytes((r, g, b, a)) * width
+    return bytearray(row * height)
+
+
+def blit_frame(
+    buffer: bytearray,
+    canvas_width: int,
+    canvas_height: int,
+    left: int,
+    top: int,
+    frame: ShapeFrame,
+    pixels: list[int],
+    palette: list[tuple[int, int, int]],
+    flipped: bool,
+) -> None:
+    for src_y in range(frame.height):
+        dst_y = top + src_y
+        if dst_y < 0 or dst_y >= canvas_height:
+            continue
+        row_base = src_y * frame.width
+        for src_x in range(frame.width):
+            color_index = pixels[row_base + (frame.width - 1 - src_x if flipped else src_x)]
+            if color_index < 0:
+                continue
+            dst_x = left + src_x
+            if dst_x < 0 or dst_x >= canvas_width:
+                continue
+            pixel_base = (dst_y * canvas_width + dst_x) * 4
+            r, g, b = palette[color_index]
+            buffer[pixel_base : pixel_base + 4] = bytes((r, g, b, 255))
+
+
+def write_png_rgba(path: Path, width: int, height: int, pixels: bytearray) -> None:
+    def chunk(chunk_type: bytes, payload: bytes) -> bytes:
+        return (
+            struct.pack(">I", len(payload))
+            + chunk_type
+            + payload
+            + struct.pack(">I", zlib.crc32(chunk_type + payload) & 0xFFFFFFFF)
+        )
+
+    rows = bytearray()
+    stride = width * 4
+    for row in range(height):
+        rows.append(0)
+        start = row * stride
+        rows.extend(pixels[start : start + stride])
+
+    payload = bytearray(b"\x89PNG\r\n\x1a\n")
+    payload.extend(chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 6, 0, 0, 0)))
+    payload.extend(chunk(b"IDAT", zlib.compress(bytes(rows), level=9)))
+    payload.extend(chunk(b"IEND", b""))
+    path.write_bytes(payload)

tools/crusader_map/sorting.py

@@ -0,0 +1,418 @@
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from typing import Callable
+
+from .formats import FLAG_FLIPPED, MapItem, ShapeArchive, ShapeFrame, ShapeInfo
+
+
+@dataclass(frozen=True)
+class InvalidRenderItem:
+    shape: int
+    frame: int
+    x: int
+    y: int
+    z: int
+    source: str
+    reason: str
+
+
+@dataclass
+class SortNode:
+    item: MapItem
+    info: ShapeInfo
+    frame: ShapeFrame
+    pixels: list[int]
+    left: int
+    top: int
+    right: int
+    bottom: int
+    x: int
+    x_left: int
+    y: int
+    y_far: int
+    z: int
+    z_top: int
+    sx_left: int
+    sx_right: int
+    sx_top: int
+    sy_top: int
+    sx_bot: int
+    sy_bot: int
+    fbigsq: bool
+    flat: bool
+    occl: bool
+    solid: bool
+    draw: bool
+    roof: bool
+    noisy: bool
+    anim: bool
+    trans: bool
+    fixed: bool
+    land: bool
+    sprite: bool
+    invitem: bool
+    occluded: bool = False
+    order: int = -1
+    depends: list["SortNode"] = field(default_factory=list)
+
+    def list_less_than(self, other: "SortNode") -> bool:
+        if self.sprite != other.sprite:
+            return self.sprite < other.sprite
+        if self.z != other.z:
+            return self.z < other.z
+        return self.flat > other.flat
+
+    def overlap(self, other: "SortNode") -> bool:
+        if not rect_intersects(self, other):
+            return False
+
+        point_top_diff = (self.sx_top - other.sx_bot, self.sy_top - other.sy_bot)
+        point_bot_diff = (self.sx_bot - other.sx_top, self.sy_bot - other.sy_top)
+
+        dot_top_left = point_top_diff[0] + point_top_diff[1] * 2
+        dot_top_right = -point_top_diff[0] + point_top_diff[1] * 2
+        dot_bot_left = point_bot_diff[0] - point_bot_diff[1] * 2
+        dot_bot_right = -point_bot_diff[0] - point_bot_diff[1] * 2
+
+        right_clear = self.sx_right <= other.sx_left
+        left_clear = self.sx_left >= other.sx_right
+        top_left_clear = dot_top_left >= 0
+        top_right_clear = dot_top_right >= 0
+        bot_left_clear = dot_bot_left >= 0
+        bot_right_clear = dot_bot_right >= 0
+
+        clear = right_clear or left_clear or (bot_right_clear or bot_left_clear) or (top_right_clear or top_left_clear)
+        return not clear
+
+    def occludes(self, other: "SortNode") -> bool:
+        if not rect_contains(self, other):
+            return False
+
+        point_top_diff = (self.sx_top - other.sx_top, self.sy_top - other.sy_top)
+        point_bot_diff = (self.sx_bot - other.sx_bot, self.sy_bot - other.sy_bot)
+
+        dot_top_left = point_top_diff[0] + point_top_diff[1] * 2
+        dot_top_right = -point_top_diff[0] + point_top_diff[1] * 2
+        dot_bot_left = point_bot_diff[0] - point_bot_diff[1] * 2
+        dot_bot_right = -point_bot_diff[0] - point_bot_diff[1] * 2
+
+        right_res = self.sx_right >= other.sx_right
+        left_res = self.sx_left <= other.sx_left
+        top_left_res = dot_top_left <= 0
+        top_right_res = dot_top_right <= 0
+        bot_left_res = dot_bot_left <= 0
+        bot_right_res = dot_bot_right <= 0
+        return right_res and left_res and bot_right_res and bot_left_res and top_right_res and top_left_res
+
+    def below(self, other: "SortNode") -> bool:
+        if self.sprite != other.sprite:
+            return self.sprite < other.sprite
+
+        if self.flat and other.flat:
+            if self.z != other.z:
+                return self.z < other.z
+        elif self.invitem == other.invitem:
+            if self.z_top <= other.z:
+                return True
+            if self.z >= other.z_top:
+                return False
+
+        y_flat_self = self.y_far == self.y
+        y_flat_other = other.y_far == other.y
+        if y_flat_self and y_flat_other:
+            if self.y // 32 != other.y // 32:
+                return self.y < other.y
+        else:
+            if self.y <= other.y_far:
+                return True
+            if self.y_far >= other.y:
+                return False
+
+        x_flat_self = self.x_left == self.x
+        x_flat_other = other.x_left == other.x
+        if x_flat_self and x_flat_other:
+            if self.x // 32 != other.x // 32:
+                return self.x < other.x
+        else:
+            if self.x <= other.x_left:
+                return True
+            if self.x_left >= other.x:
+                return False
+
+        if self.z_top - 8 <= other.z and self.z < other.z_top - 8:
+            return True
+        if self.z >= other.z_top - 8 and self.z_top - 8 > other.z:
+            return False
+
+        if y_flat_self != y_flat_other:
+            if self.y // 32 <= other.y_far // 32:
+                return True
+            if self.y_far // 32 >= other.y // 32:
+                return False
+            y_center_self = (self.y_far // 32 + self.y // 32) // 2
+            y_center_other = (other.y_far // 32 + other.y // 32) // 2
+            if y_center_self != y_center_other:
+                return y_center_self < y_center_other
+
+        if x_flat_self != x_flat_other:
+            if self.x // 32 <= other.x_left // 32:
+                return True
+            if self.x_left // 32 >= other.x // 32:
+                return False
+            x_center_self = (self.x_left // 32 + self.x // 32) // 2
+            x_center_other = (other.x_left // 32 + other.x // 32) // 2
+            if x_center_self != x_center_other:
+                return x_center_self < x_center_other
+
+        if self.flat or other.flat:
+            if self.z != other.z:
+                return self.z < other.z
+            if self.invitem != other.invitem:
+                return self.invitem < other.invitem
+            if self.flat != other.flat:
+                return self.flat > other.flat
+            if self.trans != other.trans:
+                return self.trans < other.trans
+            if self.anim != other.anim:
+                return self.anim < other.anim
+            if self.draw != other.draw:
+                return self.draw > other.draw
+            if self.solid != other.solid:
+                return self.solid > other.solid
+            if self.occl != other.occl:
+                return self.occl > other.occl
+            if self.fbigsq != other.fbigsq:
+                return self.fbigsq > other.fbigsq
+
+        if self.x == other.x and self.y == other.y and self.trans != other.trans:
+            return self.trans < other.trans
+
+        if self.land and other.land and self.roof != other.roof:
+            return self.roof < other.roof
+        if self.roof != other.roof:
+            return self.roof > other.roof
+        if self.z != other.z:
+            return self.z < other.z
+
+        if x_flat_self or x_flat_other or y_flat_self or y_flat_other:
+            if self.sx_left != other.sx_left:
+                return self.sx_left > other.sx_left
+            if self.sy_bot != other.sy_bot:
+                return self.sy_bot < other.sy_bot
+
+        if self.x + self.y != other.x + other.y:
+            return self.x + self.y < other.x + other.y
+        if self.x_left + self.y_far != other.x_left + other.y_far:
+            return self.x_left + self.y_far < other.x_left + other.y_far
+        if self.y != other.y:
+            return self.y < other.y
+        if self.x != other.x:
+            return self.x < other.x
+        if self.item.shape != other.item.shape:
+            return self.item.shape < other.item.shape
+        return self.item.frame < other.item.frame
+
+
+def rect_intersects(left: SortNode, right: SortNode) -> bool:
+    return left.left < right.right and left.right > right.left and left.top < right.bottom and left.bottom > right.top
+
+
+def rect_contains(outer: SortNode, inner: SortNode) -> bool:
+    return outer.left <= inner.left and outer.top <= inner.top and outer.right >= inner.right and outer.bottom >= inner.bottom
+
+
+def build_sort_node(item: MapItem, info: ShapeInfo, frame: ShapeFrame, pixels: list[int]) -> SortNode:
+    flipped = bool(item.flags & FLAG_FLIPPED)
+    xdim = info.y * 32 if flipped else info.x * 32
+    ydim = info.x * 32 if flipped else info.y * 32
+    zdim = info.z * 8
+
+    x = item.x
+    y = item.y
+    z = item.z
+    x_left = x - xdim
+    y_far = y - ydim
+    z_top = z + zdim
+
+    sx_left = x_left // 4 - y // 4
+    sx_right = x // 4 - y_far // 4
+    sx_top = x_left // 4 - y_far // 4
+    sy_top = x_left // 8 + y_far // 8 - z_top
+    sx_bot = x // 4 - y // 4
+    sy_bot = x // 8 + y // 8 - z
+
+    left = sx_bot + frame.xoff - frame.width if flipped else sx_bot - frame.xoff
+    top = sy_bot - frame.yoff
+    right = left + frame.width
+    bottom = top + frame.height
+
+    return SortNode(
+        item=item,
+        info=info,
+        frame=frame,
+        pixels=pixels,
+        left=left,
+        top=top,
+        right=right,
+        bottom=bottom,
+        x=x,
+        x_left=x_left,
+        y=y,
+        y_far=y_far,
+        z=z,
+        z_top=z_top,
+        sx_left=sx_left,
+        sx_right=sx_right,
+        sx_top=sx_top,
+        sy_top=sy_top,
+        sx_bot=sx_bot,
+        sy_bot=sy_bot,
+        fbigsq=xdim == ydim and xdim >= 128,
+        flat=zdim == 0,
+        occl=info.is_occl and not info.is_translucent,
+        solid=info.is_solid,
+        draw=info.is_draw,
+        roof=info.is_roof,
+        noisy=info.is_noisy,
+        anim=info.anim_type != 0,
+        trans=info.is_translucent,
+        fixed=info.is_fixed,
+        land=info.is_land,
+        sprite=False,
+        invitem=info.is_invitem,
+    )
+
+
+def insert_dependency_sorted(depends: list[SortNode], node: SortNode) -> bool:
+    for index, current in enumerate(depends):
+        if current is node:
+            return False
+        if node.list_less_than(current):
+            depends.insert(index, node)
+            return True
+    depends.append(node)
+    return True
+
+
+def resolve_paint_order(
+    ordered: list[SortNode],
+    progress: Callable[[str], None] | None = None,
+    checkpoint_every: int = 0,
+) -> list[SortNode]:
+    painted: list[SortNode] = []
+
+    def visit(node: SortNode) -> None:
+        if node.occluded or node.order >= 0:
+            return
+        node.order = -2
+        for dependency in node.depends:
+            if dependency.order == -2:
+                break
+            if dependency.order == -1:
+                visit(dependency)
+        node.order = painted[-1].order + 1 if painted else 0
+        painted.append(node)
+        if progress is not None and checkpoint_every > 0 and len(painted) % checkpoint_every == 0:
+            progress(f"paint resolved={len(painted)} of {len(ordered)}")
+
+    for node in ordered:
+        if node.order == -1:
+            visit(node)
+    if progress is not None:
+        progress(f"paint complete resolved={len(painted)} of {len(ordered)}")
+    return painted
+
+
+def prepare_sorted_items(
+    items: list[MapItem],
+    archive: ShapeArchive,
+    shape_infos: list[ShapeInfo],
+    progress: Callable[[str], None] | None = None,
+    checkpoint_every: int = 0,
+    max_invalid_details: int = 20,
+) -> tuple[int, int, int, int, list[SortNode], int, int, list[InvalidRenderItem]]:
+    ordered: list[SortNode] = []
+    min_left = sys.maxsize
+    min_top = sys.maxsize
+    max_right = -sys.maxsize
+    max_bottom = -sys.maxsize
+    occluded_count = 0
+    invalid_item_count = 0
+    invalid_items: list[InvalidRenderItem] = []
+    dependency_count = 0
+
+    for item_index, item in enumerate(items, start=1):
+        try:
+            frame, pixels = archive.decode_frame(item.shape, item.frame)
+        except (IndexError, ValueError) as error:
+            invalid_item_count += 1
+            if len(invalid_items) < max_invalid_details:
+                invalid_items.append(
+                    InvalidRenderItem(
+                        shape=item.shape,
+                        frame=item.frame,
+                        x=item.x,
+                        y=item.y,
+                        z=item.z,
+                        source=item.source,
+                        reason=str(error),
+                    )
+                )
+            continue
+        node = build_sort_node(item, shape_infos[item.shape], frame, pixels)
+
+        min_left = min(min_left, node.left)
+        min_top = min(min_top, node.top)
+        max_right = max(max_right, node.right)
+        max_bottom = max(max_bottom, node.bottom)
+
+        insert_at = len(ordered)
+        for index, other in enumerate(ordered):
+            if insert_at == len(ordered) and node.list_less_than(other):
+                insert_at = index
+            if other.occluded:
+                continue
+            if not node.overlap(other):
+                continue
+            if node.below(other):
+                if other.occl and other.occludes(node):
+                    node.occluded = True
+                    occluded_count += 1
+                    break
+                if insert_dependency_sorted(other.depends, node):
+                    dependency_count += 1
+            else:
+                if node.occl and node.occludes(other):
+                    if not other.occluded:
+                        other.occluded = True
+                        occluded_count += 1
+                else:
+                    if insert_dependency_sorted(node.depends, other):
+                        dependency_count += 1
+        ordered.insert(insert_at, node)
+        if progress is not None and checkpoint_every > 0 and item_index % checkpoint_every == 0:
+            progress(
+                "sort "
+                f"processed={item_index} valid={len(ordered)} occluded={occluded_count} invalid={invalid_item_count} "
+                f"dependencies={dependency_count}"
+            )
+
+    if progress is not None:
+        progress(
+            "sort complete "
+            f"processed={len(items)} valid={len(ordered)} occluded={occluded_count} invalid={invalid_item_count} "
+            f"dependencies={dependency_count}"
+        )
+
+    return (
+        min_left,
+        min_top,
+        max_right,
+        max_bottom,
+        resolve_paint_order(ordered, progress=progress, checkpoint_every=checkpoint_every),
+        occluded_count,
+        invalid_item_count,
+        invalid_items,
+    )

tools/render_all_maps.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import argparse
+import os
+import struct
+import subprocess
+import sys
+from pathlib import Path
+
+
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from tools.crusader_map.formats import collect_render_items, load_globs, load_map_items, load_typeflags
+
+
+def get_map_count(fixed_dat: Path) -> int:
+    data = fixed_dat.read_bytes()
+    return struct.unpack_from("<H", data, 0x54)[0]
+
+
+def has_world_rect(extra_args: list[str]) -> bool:
+    return "--world-rect" in extra_args
+
+
+def render_game(repo_root: Path, python_exe: str, game: str, start: int | None, end: int | None, extra_args: list[str]) -> int:
+    out_dir = repo_root / "out" / game
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    if game == "regret":
+        static_dir = repo_root / "STATIC_REGRET"
+    else:
+        static_dir = repo_root / "STATIC"
+
+    fixed_dat = static_dir / "FIXED.DAT"
+    if not fixed_dat.exists():
+        print(f"Missing {fixed_dat}", file=sys.stderr)
+        return 1
+
+    map_count = get_map_count(fixed_dat)
+    shape_infos = load_typeflags(static_dir / "TYPEFLAG.DAT")
+    globs = load_globs(static_dir / "GLOB.FLX")
+    batch_max_items = int(os.environ.get("BATCH_MAX_ITEMS", "0"))
+    world_rect_requested = has_world_rect(extra_args)
+    start_index = 0 if start is None else max(0, start)
+    end_index = map_count - 1 if end is None else min(end, map_count - 1)
+    if start_index > end_index:
+        print(f"Invalid map range {start_index}..{end_index} for {game} ({map_count} maps)", file=sys.stderr)
+        return 1
+
+    print(f"Rendering {game} maps {start_index}..{end_index} into {out_dir}")
+    failed = False
+    script_path = repo_root / "tools" / "render_crusader_map.py"
+    for map_index in range(start_index, end_index + 1):
+        print(f"[{game}] Rendering map {map_index}...")
+        output_png = out_dir / f"map-{map_index}.png"
+        output_json = out_dir / f"map-{map_index}.json"
+        base_items = load_map_items(fixed_dat, map_index)
+        render_items = collect_render_items(
+            base_items,
+            shape_infos,
+            globs,
+            include_editor=True,
+            expand_globs=True,
+            world_rect=None,
+            include_roofs=False,
+            include_hidden_markers=True,
+        )
+        if not render_items:
+            print(f"[{game}] Skipping empty map {map_index}.")
+            output_png.unlink(missing_ok=True)
+            output_json.unlink(missing_ok=True)
+            continue
+        if batch_max_items > 0 and not world_rect_requested and len(render_items) > batch_max_items:
+            print(
+                f"[{game}] Skipping map {map_index}: {len(render_items)} render items exceed batch threshold {batch_max_items}. "
+                "Set BATCH_MAX_ITEMS=0 to disable or use RENDER_ARGS=--world-rect ... for bounded runs.",
+                file=sys.stderr,
+            )
+            output_png.unlink(missing_ok=True)
+            output_json.unlink(missing_ok=True)
+            continue
+        command = [
+            python_exe,
+            str(script_path),
+            "--game",
+            game,
+            "--map",
+            str(map_index),
+            "--output",
+            str(output_png),
+            "--metadata",
+            str(output_json),
+            *extra_args,
+        ]
+        result = subprocess.run(command, cwd=repo_root)
+        if result.returncode != 0:
+            print(f"[{game}] Map {map_index} failed.", file=sys.stderr)
+            failed = True
+    return 1 if failed else 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Render every Crusader fixed map for one or both games.")
+    parser.add_argument("--game", choices=("remorse", "regret", "all"), required=True)
+    parser.add_argument("--start", type=int, help="Optional starting map index.")
+    parser.add_argument("--end", type=int, help="Optional ending map index.")
+    args, extra_args = parser.parse_known_args()
+
+    repo_root = Path(__file__).resolve().parents[1]
+    python_exe = os.environ.get("PYTHON_EXE") or sys.executable
+
+    games = [args.game] if args.game != "all" else ["remorse", "regret"]
+    exit_code = 0
+    for game in games:
+        exit_code |= render_game(repo_root, python_exe, game, args.start, args.end, extra_args)
+    return exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())