import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import sharp from "sharp"; import { TILE_CACHE_ROOT, TILE_SIZE } from "../config.js"; import { EGG_FAMILIES, FLAG_INVISIBLE, FLAG_FLIPPED, ShapeArchive, collectRenderItems, loadGlobs, loadMapItems, loadPalette, loadTypeflags, resolveStaticFile, summarizeRenderClasses } from "./formats.js"; import { blitFrame, encodePng, rgbaBuffer } from "./png.js"; import { prepareSortedItems, projectItemGeometry } from "./sorting.js"; function nowIso() { return new Date().toISOString(); } function ensureDir(dirPath) { fs.mkdirSync(dirPath, { recursive: true }); } const DOWNLOAD_CACHE_ROOT = path.join(TILE_CACHE_ROOT, "downloads"); const RENDER_CACHE_VERSION = "v2-overlays-as-sprites"; sharp.cache(false); function normalizeBuildOptions(options = {}) { return { includeEditor: options.includeEditor !== false, includeRoofs: options.includeRoofs === true }; } function buildOptionSuffix(options) { return `${RENDER_CACHE_VERSION}_editor-${options.includeEditor ? "on" : "off"}_roofs-${options.includeRoofs ? "on" : "off"}`; } function toHex(value, width = 4) { return `0x${value.toString(16).padStart(width, "0")}`; } function createEmptyProjection() { return { projected: [], invalidItemCount: 0, invalidItems: [] }; } function computeBoundsFromNodes(nodes) { if (!nodes.length) { return null; } let minLeft = Number.MAX_SAFE_INTEGER; let minTop = Number.MAX_SAFE_INTEGER; let maxRight = -Number.MAX_SAFE_INTEGER; let maxBottom = -Number.MAX_SAFE_INTEGER; for (const node of nodes) { minLeft = Math.min(minLeft, node.left); minTop = Math.min(minTop, node.top); maxRight = Math.max(maxRight, node.right); maxBottom = Math.max(maxBottom, node.bottom); } return { minLeft, minTop, maxRight, maxBottom, width: maxRight - minLeft, height: maxBottom - minTop }; } function mergeBounds(boundsList) { const validBounds = boundsList.filter(Boolean); if (!validBounds.length) { return null; } const minLeft = Math.min(...validBounds.map((bounds) => bounds.minLeft)); const minTop = Math.min(...validBounds.map((bounds) => bounds.minTop)); const maxRight = Math.max(...validBounds.map((bounds) => bounds.maxRight)); const maxBottom = Math.max(...validBounds.map((bounds) => bounds.maxBottom)); return { minLeft, minTop, maxRight, maxBottom, width: maxRight - minLeft, height: maxBottom - minTop }; } function isOverlayItem(item, shapeInfos) { return item.shape < shapeInfos.length && shapeInfos[item.shape].isEditor; } function splitRenderItems(renderItems, shapeInfos, includeEditor) { if (!includeEditor) { return { baseItems: renderItems, overlayItems: [] }; } const baseItems = []; const overlayItems = []; for (const item of renderItems) { if (isOverlayItem(item, shapeInfos)) { overlayItems.push(item); } else { baseItems.push(item); } } return { baseItems, overlayItems }; } function classifyOverlayKind(item, info) { if ((item.flags & FLAG_INVISIBLE) || info.isOccl || info.isInvitem) { return "helper"; } if (EGG_FAMILIES.has(info.family)) { return "egg"; } if (info.isRoof) { return "roof"; } return "editor"; } function overlayLabel(kind) { if (kind === "helper") { return "Helper Geometry"; } if (kind === "egg") { return "Egg Trigger"; } if (kind === "roof") { return "Roof Marker"; } return "Editor Object"; } function overlayNotes(item, info) { const notes = []; if (item.flags & FLAG_INVISIBLE) { notes.push("invisible-flagged"); } if (info.isOccl) { notes.push("occluding-geometry"); } if (info.isInvitem) { notes.push("invitem-family"); } if (EGG_FAMILIES.has(info.family)) { notes.push("egg-family"); } if (info.isRoof) { notes.push("roof-flagged"); } if (info.isTranslucent) { notes.push("translucent"); } return notes; } function classifyBaseKind(info) { if (info.isRoof) { return "roof"; } if (info.isOccl || info.isInvitem) { return "helper"; } if (info.isLand) { return "terrain"; } return "base"; } function inspectLabel(layer, kind) { if (layer === "overlay") { return overlayLabel(kind); } if (kind === "roof") { return "Roof Shape"; } if (kind === "helper") { return "Occluding Helper"; } if (kind === "terrain") { return "Terrain Shape"; } return "Map Shape"; } function serializeInspectableItem(node, minLeft, minTop, id, layer, stackOrder) { const { item, info, frame } = node; const kind = layer === "overlay" ? classifyOverlayKind(item, info) : classifyBaseKind(info); const sceneLeft = node.left - minLeft; const sceneTop = node.top - minTop; return { id, layer, stackOrder, kind, label: inspectLabel(layer, kind), family: info.family, source: item.source, world: { x: item.x, y: item.y, z: item.z }, mapNum: item.mapNum, npcNum: item.npcNum, nextItem: item.nextItem, quality: item.quality, shape: item.shape, frame: item.frame, frameSize: { width: frame.width, height: frame.height, xoff: frame.xoff, yoff: frame.yoff }, screen: { left: sceneLeft, top: sceneTop, right: node.right - minLeft, bottom: node.bottom - minTop, width: node.right - node.left, height: node.bottom - node.top, anchorX: Math.trunc(sceneLeft + (node.right - node.left) / 2), anchorY: node.bottom - minTop }, flags: { raw: item.flags, hex: toHex(item.flags), invisible: Boolean(item.flags & FLAG_INVISIBLE), flipped: Boolean(item.flags & FLAG_FLIPPED) }, traits: { editor: info.isEditor, roof: info.isRoof, occluding: info.isOccl, translucent: info.isTranslucent, solid: info.isSolid, fixed: info.isFixed, land: info.isLand, draw: info.isDraw, invitem: info.isInvitem, animType: info.animType }, notes: overlayNotes(item, info) }; } function serializeOverlayItem(node, minLeft, minTop, index) { const { item, info, frame } = node; const kind = classifyOverlayKind(item, info); const sceneLeft = node.left - minLeft; const sceneTop = node.top - minTop; const screenWidth = node.right - node.left; const screenHeight = node.bottom - node.top; return { id: `${index}:${item.source}:${item.shape}:${item.frame}:${item.x}:${item.y}:${item.z}`, index, kind, label: overlayLabel(kind), family: info.family, source: item.source, world: { x: item.x, y: item.y, z: item.z }, mapNum: item.mapNum, npcNum: item.npcNum, nextItem: item.nextItem, quality: item.quality, shape: item.shape, frame: item.frame, frameSize: { width: frame.width, height: frame.height, xoff: frame.xoff, yoff: frame.yoff }, screen: { left: sceneLeft, top: sceneTop, right: node.right - minLeft, bottom: node.bottom - minTop, width: screenWidth, height: screenHeight, anchorX: Math.trunc(sceneLeft + screenWidth / 2), anchorY: node.bottom - minTop }, flags: { raw: item.flags, hex: toHex(item.flags), invisible: Boolean(item.flags & FLAG_INVISIBLE), flipped: Boolean(item.flags & FLAG_FLIPPED) }, traits: { editor: info.isEditor, roof: info.isRoof, occluding: info.isOccl, translucent: info.isTranslucent, solid: info.isSolid, fixed: info.isFixed, land: info.isLand, draw: info.isDraw, invitem: info.isInvitem, animType: info.animType }, presentation: { opacity: kind === "helper" ? 0.5 : info.isTranslucent ? 0.7 : 1 }, notes: overlayNotes(item, info) }; } function summarizeOverlayItems(items) { const kindCounts = {}; const familyCounts = {}; const sourceCounts = {}; for (const item of items) { kindCounts[item.kind] = (kindCounts[item.kind] ?? 0) + 1; familyCounts[item.family] = (familyCounts[item.family] ?? 0) + 1; sourceCounts[item.source] = (sourceCounts[item.source] ?? 0) + 1; } const topFamilies = Object.entries(familyCounts) .sort((left, right) => right[1] - left[1] || Number(left[0]) - Number(right[0])) .slice(0, 6) .map(([family, count]) => ({ family: Number(family), count })); return { itemCount: items.length, kindCounts, sourceCounts, topFamilies, helperCount: kindCounts.helper ?? 0 }; } function makeUsageInfo(gameId, mapId, baseItems, renderItems) { const itemMapNums = [...new Set(baseItems.map((item) => item.mapNum))].sort((left, right) => left - right); return { status: "unknown", confidence: "unknown", knownHints: [], itemMapNums, nonzeroItemMapNums: itemMapNums.filter((value) => value !== 0), npcLinkedItemCount: baseItems.filter((item) => item.npcNum !== 0).length, note: "No authoritative mission-to-map ownership table has been extracted yet. Unknown does not imply orphaned.", hasRenderableContent: renderItems.length > 0, game: gameId, map: mapId }; } function buildEmptyMetadata(gameConfig, mapId, baseItems, reason) { return { game: gameConfig.id, gameLabel: gameConfig.label, map: mapId, rawItemCount: baseItems.length, itemCount: 0, baseRasterItemCount: 0, overlayItemCount: 0, paintedItemCount: 0, occludedItemCount: 0, invalidItemCount: 0, invalidItems: [], overlaySummary: summarizeOverlayItems([]), usage: makeUsageInfo(gameConfig.id, mapId, baseItems, []), baseItemSummary: { roofItems: 0, editorItems: 0, eggFamilyItems: 0, invisibleFlaggedItems: 0, npcLinkedItems: 0 }, sorter: "scummvm_dependency_graph", isEmpty: true, emptyReason: reason, filters: { includeEditor: true, includeRoofs: false }, bounds: { screenLeft: 0, screenTop: 0, screenRight: TILE_SIZE, screenBottom: TILE_SIZE, width: TILE_SIZE, height: TILE_SIZE }, tileSize: TILE_SIZE, tileCountX: 1, tileCountY: 1, zoom: { min: 0.01, max: 8, step: 0.1, initial: 1 } }; } export class BuildManager { constructor(catalog) { this.catalog = catalog; this.assetCache = new Map(); this.jobs = new Map(); this.jobsByKey = new Map(); this.tileCache = new Map(); ensureDir(TILE_CACHE_ROOT); } listCatalog() { return this.catalog; } getJob(jobId) { return this.jobs.get(jobId) ?? null; } async createOrReuseBuild(gameConfig, mapId, rawOptions = {}) { const options = normalizeBuildOptions(rawOptions); const key = `${gameConfig.id}:${mapId}:${buildOptionSuffix(options)}`; const existing = this.jobsByKey.get(key); if (existing) { if (existing.status === "ready" || existing.status === "building") { return existing; } this.jobsByKey.delete(key); } const job = { id: crypto.randomUUID(), key, game: gameConfig.id, mapId, options, status: "queued", phase: "queued", createdAt: nowIso(), updatedAt: nowIso(), progress: [], error: null, metadata: null, build: null }; this.jobs.set(job.id, job); this.jobsByKey.set(key, job); void this.runBuild(job, gameConfig); return job; } async runBuild(job, gameConfig) { try { job.status = "building"; job.phase = "loading-assets"; this.touchJob(job, `Loading ${gameConfig.label} assets`); const assets = this.getAssets(gameConfig); job.phase = "loading-map"; this.touchJob(job, `Loading map ${job.mapId}`); const fixedDatPath = resolveStaticFile(gameConfig.staticDir, "FIXED.DAT"); const baseItems = loadMapItems(fixedDatPath, job.mapId); this.touchJob(job, `Loaded ${baseItems.length} fixed records`); job.phase = "collecting-items"; const renderItems = collectRenderItems(baseItems, assets.shapeInfos, assets.globs, { includeEditor: job.options.includeEditor, expandGlobs: true, worldRect: null, includeRoofs: job.options.includeRoofs, includeHiddenMarkers: true, checkpointEvery: 2000, progress: (message) => this.touchJob(job, message) }); if (!renderItems.length) { job.build = { assets, prepared: [], overlays: [], overlayNodes: [], overlayNodeById: new Map(), inspectables: [], minLeft: 0, minTop: 0, width: TILE_SIZE, height: TILE_SIZE }; job.metadata = buildEmptyMetadata(gameConfig, job.mapId, baseItems, "This map has no renderable items in FIXED.DAT."); job.metadata.filters = { includeEditor: job.options.includeEditor, includeRoofs: job.options.includeRoofs }; job.status = "ready"; job.phase = "ready"; this.touchJob(job, "Build ready: map is empty, serving a blank placeholder tile"); return; } const splitItems = splitRenderItems(renderItems, assets.shapeInfos, job.options.includeEditor); this.touchJob( job, `Split ${splitItems.baseItems.length} base items and ${splitItems.overlayItems.length} overlay items` ); job.phase = "sorting"; const sorted = splitItems.baseItems.length ? prepareSortedItems(splitItems.baseItems, assets.shapeArchive, assets.shapeInfos, { checkpointEvery: 2000, maxInvalidDetails: 20, progress: (message) => this.touchJob(job, message) }) : { minLeft: 0, minTop: 0, maxRight: TILE_SIZE, maxBottom: TILE_SIZE, prepared: [], occludedCount: 0, invalidItemCount: 0, invalidItems: [] }; const overlayProjection = splitItems.overlayItems.length ? projectItemGeometry(splitItems.overlayItems, assets.shapeArchive, assets.shapeInfos, { checkpointEvery: 2000, maxInvalidDetails: 20, progress: (message) => this.touchJob(job, message) }) : createEmptyProjection(); if (!sorted.prepared.length && !overlayProjection.projected.length) { job.build = { assets, prepared: [], overlays: [], overlayNodes: [], overlayNodeById: new Map(), inspectables: [], minLeft: 0, minTop: 0, width: TILE_SIZE, height: TILE_SIZE }; job.metadata = buildEmptyMetadata( gameConfig, job.mapId, baseItems, "This map resolved to no valid shape or frame pairs after decoding." ); job.metadata.filters = { includeEditor: job.options.includeEditor, includeRoofs: job.options.includeRoofs }; job.status = "ready"; job.phase = "ready"; this.touchJob(job, "Build ready: no valid frames were renderable, serving a blank placeholder tile"); return; } const bounds = mergeBounds([ sorted.prepared.length ? { minLeft: sorted.minLeft, minTop: sorted.minTop, maxRight: sorted.maxRight, maxBottom: sorted.maxBottom, width: sorted.maxRight - sorted.minLeft, height: sorted.maxBottom - sorted.minTop } : null, computeBoundsFromNodes(overlayProjection.projected) ]); if (!bounds || bounds.width <= 0 || bounds.height <= 0) { throw new Error("Computed image bounds are invalid"); } const overlays = overlayProjection.projected.map((node, index) => serializeOverlayItem(node, bounds.minLeft, bounds.minTop, index) ); const overlayNodeById = new Map(); for (let index = 0; index < overlays.length; index += 1) { overlayNodeById.set(overlays[index].id, overlayProjection.projected[index]); } const inspectables = []; for (let index = 0; index < sorted.prepared.length; index += 1) { const node = sorted.prepared[index]; const stackOrder = node.order >= 0 ? node.order : index; inspectables.push( serializeInspectableItem( node, bounds.minLeft, bounds.minTop, `base:${stackOrder}:${node.item.source}:${node.item.shape}:${node.item.frame}:${node.item.x}:${node.item.y}:${node.item.z}`, "base", stackOrder ) ); } for (let index = 0; index < overlays.length; index += 1) { inspectables.push( serializeInspectableItem( overlayProjection.projected[index], bounds.minLeft, bounds.minTop, `overlay:${overlays[index].id}`, "overlay", sorted.prepared.length + index ) ); } inspectables.sort((left, right) => left.stackOrder - right.stackOrder); const invalidItemCount = sorted.invalidItemCount + overlayProjection.invalidItemCount; const invalidItems = [...sorted.invalidItems, ...overlayProjection.invalidItems].slice(0, 20); const metadata = { game: gameConfig.id, gameLabel: gameConfig.label, map: job.mapId, rawItemCount: baseItems.length, itemCount: renderItems.length, baseRasterItemCount: splitItems.baseItems.length, overlayItemCount: overlays.length, paintedItemCount: sorted.prepared.length, occludedItemCount: sorted.occludedCount, invalidItemCount, invalidItems, overlaySummary: summarizeOverlayItems(overlays), usage: makeUsageInfo(gameConfig.id, job.mapId, baseItems, renderItems), baseItemSummary: summarizeRenderClasses(baseItems, assets.shapeInfos), sorter: "scummvm_dependency_graph", isEmpty: false, emptyReason: null, filters: { includeEditor: job.options.includeEditor, includeRoofs: job.options.includeRoofs }, bounds: { screenLeft: bounds.minLeft, screenTop: bounds.minTop, screenRight: bounds.maxRight, screenBottom: bounds.maxBottom, width: bounds.width, height: bounds.height }, tileSize: TILE_SIZE, tileCountX: Math.ceil(bounds.width / TILE_SIZE), tileCountY: Math.ceil(bounds.height / TILE_SIZE), zoom: { min: 0.01, max: 8, step: 0.1, initial: 1 } }; job.build = { assets, prepared: sorted.prepared, overlays, overlayNodes: overlayProjection.projected, overlayNodeById, inspectables, minLeft: bounds.minLeft, minTop: bounds.minTop, width: bounds.width, height: bounds.height }; job.metadata = metadata; job.status = "ready"; job.phase = "ready"; this.touchJob(job, `Build ready with ${metadata.tileCountX}x${metadata.tileCountY} tiles`); } catch (error) { job.status = "failed"; job.phase = "failed"; job.error = error instanceof Error ? error.message : String(error); this.touchJob(job, `Build failed: ${job.error}`); } } getAssets(gameConfig) { if (this.assetCache.has(gameConfig.id)) { return this.assetCache.get(gameConfig.id); } const assets = { palette: loadPalette(resolveStaticFile(gameConfig.staticDir, "GAMEPAL.PAL")), shapeInfos: loadTypeflags(resolveStaticFile(gameConfig.staticDir, "TYPEFLAG.DAT")), globs: loadGlobs(resolveStaticFile(gameConfig.staticDir, "GLOB.FLX")), shapeArchive: new ShapeArchive(resolveStaticFile(gameConfig.staticDir, "SHAPES.FLX")) }; this.assetCache.set(gameConfig.id, assets); return assets; } touchJob(job, message) { job.updatedAt = nowIso(); job.progress.push({ at: job.updatedAt, phase: job.phase, message }); if (job.progress.length > 100) { job.progress.splice(0, job.progress.length - 100); } } getPublicJob(job) { return { id: job.id, game: job.game, mapId: job.mapId, options: job.options, status: job.status, phase: job.phase, createdAt: job.createdAt, updatedAt: job.updatedAt, error: job.error, metadata: job.status === "ready" ? job.metadata : null, progress: job.progress }; } getMetadata(jobId, gameId, mapId) { const job = this.requireReadyJob(jobId, gameId, mapId); return job.metadata; } getOverlayData(jobId, gameId, mapId) { const job = this.requireReadyJob(jobId, gameId, mapId); return { items: job.build.overlays, summary: job.metadata.overlaySummary }; } getInspectData(jobId, gameId, mapId) { const job = this.requireReadyJob(jobId, gameId, mapId); return { items: job.build.inspectables }; } async renderOverlaySprite(jobId, gameId, mapId, overlayId, format = "webp") { const job = this.requireReadyJob(jobId, gameId, mapId); const overlay = job.build.overlays.find((item) => item.id === overlayId); const node = job.build.overlayNodeById.get(overlayId); if (!overlay || !node) { throw new Error("Unknown overlay id"); } const extension = format === "png" ? "png" : "webp"; const spritePath = path.join( TILE_CACHE_ROOT, gameId, `map-${mapId}`, buildOptionSuffix(job.options), "overlays", `${overlay.index}.${extension}` ); if (fs.existsSync(spritePath)) { return fs.readFileSync(spritePath); } const spriteWidth = Math.max(1, overlay.screen.width); const spriteHeight = Math.max(1, overlay.screen.height); const buffer = rgbaBuffer(spriteWidth, spriteHeight, [0, 0, 0, 0]); blitFrame( buffer, spriteWidth, spriteHeight, 0, 0, node.frame, node.pixels, job.build.assets.palette, Boolean(node.item.flags & FLAG_FLIPPED) ); let output; if (format === "png") { output = encodePng(spriteWidth, spriteHeight, buffer); } else { output = await sharp(buffer, { raw: { width: spriteWidth, height: spriteHeight, channels: 4 }, limitInputPixels: false }) .webp({ lossless: true, effort: 4 }) .toBuffer(); } ensureDir(path.dirname(spritePath)); fs.writeFileSync(spritePath, output); return output; } async renderFullMap(jobId, gameId, mapId) { const job = this.requireReadyJob(jobId, gameId, mapId); const outputPath = path.join( DOWNLOAD_CACHE_ROOT, gameId, `map-${mapId}`, `${gameId}-map-${mapId}-${buildOptionSuffix(job.options)}.png` ); if (fs.existsSync(outputPath)) { return outputPath; } ensureDir(path.dirname(outputPath)); const composites = []; for (let tileY = 0; tileY < job.metadata.tileCountY; tileY += 1) { for (let tileX = 0; tileX < job.metadata.tileCountX; tileX += 1) { composites.push({ input: await this.renderTile(jobId, gameId, mapId, tileX, tileY, "png"), left: tileX * job.metadata.tileSize, top: tileY * job.metadata.tileSize }); } } await sharp({ create: { width: job.metadata.bounds.width, height: job.metadata.bounds.height, channels: 4, background: { r: 10, g: 12, b: 18, alpha: 1 } }, limitInputPixels: false }) .composite(composites) .png() .toFile(outputPath); return outputPath; } async renderTile(jobId, gameId, mapId, tileX, tileY, format = "webp") { const job = this.requireReadyJob(jobId, gameId, mapId); const tileKey = `${job.id}:${format}:${tileX}:${tileY}`; if (this.tileCache.has(tileKey)) { return this.tileCache.get(tileKey); } const extension = format === "png" ? "png" : "webp"; const tilePath = path.join( TILE_CACHE_ROOT, gameId, `map-${mapId}`, buildOptionSuffix(job.options), `${tileX}-${tileY}.${extension}` ); if (fs.existsSync(tilePath)) { const cached = fs.readFileSync(tilePath); this.tileCache.set(tileKey, cached); return cached; } const tileLeft = tileX * TILE_SIZE; const tileTop = tileY * TILE_SIZE; const tileWidth = Math.max(0, Math.min(TILE_SIZE, job.build.width - tileLeft)); const tileHeight = Math.max(0, Math.min(TILE_SIZE, job.build.height - tileTop)); if (tileWidth <= 0 || tileHeight <= 0) { throw new Error("Requested tile is outside the rendered map bounds"); } const buffer = rgbaBuffer(tileWidth, tileHeight); const screenLeft = job.build.minLeft + tileLeft; const screenTop = job.build.minTop + tileTop; const screenRight = screenLeft + tileWidth; const screenBottom = screenTop + tileHeight; for (const node of job.build.prepared) { if (node.right <= screenLeft || node.left >= screenRight || node.bottom <= screenTop || node.top >= screenBottom) { continue; } blitFrame( buffer, tileWidth, tileHeight, node.left - screenLeft, node.top - screenTop, node.frame, node.pixels, job.build.assets.palette, Boolean(node.item.flags & FLAG_FLIPPED) ); } let output; if (format === "png") { output = encodePng(tileWidth, tileHeight, buffer); } else { output = await sharp(buffer, { raw: { width: tileWidth, height: tileHeight, channels: 4 }, limitInputPixels: false }) .webp({ lossless: true, effort: 4 }) .toBuffer(); } ensureDir(path.dirname(tilePath)); fs.writeFileSync(tilePath, output); this.tileCache.set(tileKey, output); return output; } requireReadyJob(jobId, gameId, mapId) { const job = this.getJob(jobId); if (!job) { throw new Error("Unknown build id"); } if (job.game !== gameId || job.mapId !== mapId) { throw new Error("Build id does not match the requested map"); } if (job.status !== "ready") { throw new Error("Build is not ready yet"); } return job; } }