From 32db61c381b4fb1a9139a16b3f3572c738f586fa Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Tue, 19 May 2026 23:29:38 +0200 Subject: [PATCH] feat(audit): implement audit logging for search requests and results - Added configuration options for audit database path and retention days in backend and processor. - Integrated audit logging in server and worker processes to track search requests, completions, and failures. - Created utility functions for reading and parsing audit logs in end-to-end tests. - Updated Docker Compose files to include audit database configuration. - Added new tests to verify audit log entries for successful and no-results searches. --- faceai/.env.example | 4 +- faceai/README.md | 17 +- faceai/apps/backend/src/audit-store.js | 472 ++++++++++++++++++++++ faceai/apps/backend/src/config.js | 2 + faceai/apps/backend/src/server.js | 88 +++- faceai/apps/processor/src/config.js | 2 + faceai/apps/processor/src/worker-utils.js | 2 +- faceai/apps/processor/src/worker.js | 35 +- faceai/docker-compose.override.yml | 4 +- faceai/docker-compose.yml | 4 + faceai/tests/e2e/faceai-simulator.spec.js | 140 ++++++- faceai/tests/e2e/faceai-test-utils.js | 181 ++++++++- faceai/tmp/faceai-no-results-smoke.json | 128 ++++++ stacks/faceai.yml | 4 + 14 files changed, 1067 insertions(+), 16 deletions(-) create mode 100644 faceai/apps/backend/src/audit-store.js create mode 100644 faceai/tmp/faceai-no-results-smoke.json diff --git a/faceai/.env.example b/faceai/.env.example index 3db21884..84f33c04 100644 --- a/faceai/.env.example +++ b/faceai/.env.example @@ -9,7 +9,7 @@ FACEAI_REDIS_IMAGE=redis:7-alpine FACEAI_CLIENT_CONTAINER_NAME=regalami-faceai FACEAI_PROCESSOR_CONTAINER_NAME=regalami-faceai-processor FACEAI_REDIS_CONTAINER_NAME=regalami-faceai-redis -FACEAI_CLIENT_DEV_IMAGE=node:20-alpine +FACEAI_CLIENT_DEV_IMAGE=node:22-trixie-slim FACEAI_PROCESSOR_DEV_IMAGE=regalami-faceai-processor-local FACEAI_PORT=3001 FACEAI_PUBLISHED_PORT=3001 @@ -28,6 +28,8 @@ FACEAI_QUEUE_NAME=faceai-searches FACEAI_RUNTIME_ROOT=/data/runtime FACEAI_UPLOAD_ROOT=/data/runtime/uploads FACEAI_LOG_ROOT=/data/logs +FACEAI_AUDIT_DB_PATH=/data/logs/faceai-audit.sqlite +FACEAI_AUDIT_RETENTION_DAYS=730 FACEAI_PKL_ROOT=/data/pkl FACEAI_RUNTIME_BIND=/mnt/storage/data/faceai/runtime FACEAI_LOG_BIND=/mnt/storage/data/faceai/logs diff --git a/faceai/README.md b/faceai/README.md index 4dd5cb9c..50f9ead5 100644 --- a/faceai/README.md +++ b/faceai/README.md @@ -103,11 +103,22 @@ After `docker compose --env-file .env.development up --build`, inspect: - `faceai/logs/backend.log` for backend startup and API-side failures - `faceai/logs/processor.log` for worker startup, queue processing, and uncaught processor errors +- `faceai/logs/faceai-audit.sqlite` for the structured 24-month audit trail of FaceAI usage - `faceai/logs/searches//worker.log` for the per-search processor trace - `faceai/logs/searches//matcher.log` for the native `face_matcher` output This keeps the useful processor diagnostics outside the Docker-managed runtime volume so they survive container rebuilds and can be inspected directly from the workspace. +The audit database is a lightweight SQLite file shared by the public FaceAI service and the processor on the existing log volume. Each search row stores the requesting user, race metadata, request timestamp, request IP and user agent, the uploaded selfie SHA-256 fingerprint, and the final match snapshot. That makes the log queryable without introducing another service and lets you recover the same result set a user originally saw by looking up `selfie_sha256`. + +The default retention is 730 days, matching the requested 24-month window. Old audit rows are pruned automatically by FaceAI during normal runtime. + +Example query: + +```bash +sqlite3 faceai/logs/faceai-audit.sqlite "SELECT search_id, user_id, race_id, requested_at, match_count FROM faceai_audit_searches WHERE selfie_sha256 = '...';" +``` + Because the service entrypoints now mirror output instead of redirecting it away, the same startup and runtime messages are also visible through `docker logs regalami-faceai`, `docker logs regalami-faceai-processor`, and Portainer's container log viewer. The current bundled Linux `face_matcher` binary is a PyInstaller build that requires `GLIBC_2.38` or newer and the `libxcb.so.1` runtime library. The checked-in local processor image satisfies that requirement. @@ -307,6 +318,8 @@ Shared application settings: | `FACEAI_QUEUE_NAME` | optional | `faceai-searches` | BullMQ queue name | | `FACEAI_RUNTIME_ROOT` | yes | `/data/runtime` | shared writable runtime root between site and processor | | `FACEAI_LOG_ROOT` | recommended | `/data/logs` | persistent host-mounted diagnostics root for backend, processor, and per-search logs | +| `FACEAI_AUDIT_DB_PATH` | recommended | `/data/logs/faceai-audit.sqlite` | SQLite audit database shared by backend and processor | +| `FACEAI_AUDIT_RETENTION_DAYS` | recommended | `730` | how long structured audit rows are kept before automatic pruning | | `FACEAI_SHARED_SECRET` | yes | long random secret | trust boundary between FaceAI and the legacy bridge | Public site settings: @@ -328,7 +341,7 @@ Processor settings: | --- | --- | --- | --- | | `FACEAI_PKL_ROOT` | yes | `/data/pkl` | mounted race-to-PKL dataset root | | `FACEAI_MATCHER_BINARY` | yes | `/app/bin/face_matcher` | matcher executable baked into the processor image | -| `FACEAI_MATCHER_TOLERANCE` | optional | `0.5` | forwarded to `face_matcher --tollerance`; must stay between `0.35` and `0.75` | +| `FACEAI_MATCHER_TOLERANCE` | optional | `0.5` | forwarded to `face_matcher --tolerance`; must stay between `0.35` and `0.75` | | `FACEAI_WORKER_CONCURRENCY` | optional | `2` | BullMQ worker concurrency | | `FACEAI_WORKER_TIMEOUT_MS` | optional | `300000` | matcher timeout in milliseconds | @@ -431,6 +444,8 @@ FACEAI_QUEUE_NAME=faceai-searches FACEAI_RUNTIME_ROOT=/data/runtime FACEAI_UPLOAD_ROOT=/data/runtime/uploads FACEAI_LOG_ROOT=/data/logs +FACEAI_AUDIT_DB_PATH=/data/logs/faceai-audit.sqlite +FACEAI_AUDIT_RETENTION_DAYS=730 FACEAI_PKL_ROOT=/data/pkl FACEAI_MATCHER_BINARY=/app/bin/face_matcher FACEAI_MATCHER_TOLERANCE=0.5 diff --git a/faceai/apps/backend/src/audit-store.js b/faceai/apps/backend/src/audit-store.js new file mode 100644 index 00000000..0daf02e1 --- /dev/null +++ b/faceai/apps/backend/src/audit-store.js @@ -0,0 +1,472 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +const ONE_DAY_MS = 24 * 60 * 60 * 1000; + +function textOrNull(value) { + if (value === undefined || value === null) { + return null; + } + + const normalized = String(value).trim(); + return normalized === '' ? null : normalized; +} + +function jsonOrNull(value) { + if (value === undefined || value === null) { + return null; + } + + return JSON.stringify(value); +} + +function openDatabase(dbPath) { + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + + const db = new DatabaseSync(dbPath); + db.exec(` + PRAGMA foreign_keys = ON; + PRAGMA busy_timeout = 5000; + PRAGMA journal_mode = WAL; + + CREATE TABLE IF NOT EXISTS faceai_audit_searches ( + search_id TEXT PRIMARY KEY, + requested_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + completed_at INTEGER, + redirect_issued_at INTEGER, + status TEXT NOT NULL, + completion_code TEXT, + error_code TEXT, + error_message TEXT, + user_id TEXT NOT NULL, + user_display_name TEXT, + user_membership_status TEXT, + race_id TEXT NOT NULL, + race_name TEXT, + race_storage TEXT, + lang TEXT, + request_ip TEXT, + request_user_agent TEXT, + return_url TEXT, + selfie_name TEXT, + selfie_sha256 TEXT, + selfie_size_bytes INTEGER, + upload_path TEXT, + result_id TEXT, + match_count INTEGER NOT NULL DEFAULT 0, + matches_json TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_faceai_audit_searches_requested_at + ON faceai_audit_searches (requested_at); + CREATE INDEX IF NOT EXISTS idx_faceai_audit_searches_user_id + ON faceai_audit_searches (user_id, requested_at DESC); + CREATE INDEX IF NOT EXISTS idx_faceai_audit_searches_race_id + ON faceai_audit_searches (race_id, requested_at DESC); + CREATE INDEX IF NOT EXISTS idx_faceai_audit_searches_selfie_sha256 + ON faceai_audit_searches (selfie_sha256, requested_at DESC); + + CREATE TABLE IF NOT EXISTS faceai_audit_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + search_id TEXT REFERENCES faceai_audit_searches(search_id) ON DELETE CASCADE, + event_type TEXT NOT NULL, + happened_at INTEGER NOT NULL, + status TEXT, + user_id TEXT, + user_display_name TEXT, + race_id TEXT, + race_name TEXT, + race_storage TEXT, + request_ip TEXT, + request_user_agent TEXT, + selfie_sha256 TEXT, + result_id TEXT, + match_count INTEGER, + completion_code TEXT, + error_code TEXT, + payload_json TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_faceai_audit_events_search_id + ON faceai_audit_events (search_id, happened_at DESC); + CREATE INDEX IF NOT EXISTS idx_faceai_audit_events_type + ON faceai_audit_events (event_type, happened_at DESC); + CREATE INDEX IF NOT EXISTS idx_faceai_audit_events_user_id + ON faceai_audit_events (user_id, happened_at DESC); + `); + + return db; +} + +export function createAuditStore({ dbPath, retentionDays }) { + const retentionMs = Math.max(1, Number(retentionDays || 730)) * ONE_DAY_MS; + const db = openDatabase(dbPath); + let lastPrunedAt = 0; + + const upsertSearchRequestStatement = db.prepare(` + INSERT INTO faceai_audit_searches ( + search_id, + requested_at, + updated_at, + status, + user_id, + user_display_name, + user_membership_status, + race_id, + race_name, + race_storage, + lang, + request_ip, + request_user_agent, + return_url, + selfie_name, + selfie_sha256, + selfie_size_bytes, + upload_path, + match_count + ) VALUES ( + @search_id, + @requested_at, + @updated_at, + @status, + @user_id, + @user_display_name, + @user_membership_status, + @race_id, + @race_name, + @race_storage, + @lang, + @request_ip, + @request_user_agent, + @return_url, + @selfie_name, + @selfie_sha256, + @selfie_size_bytes, + @upload_path, + @match_count + ) + ON CONFLICT(search_id) DO UPDATE SET + updated_at = excluded.updated_at, + status = excluded.status, + user_id = excluded.user_id, + user_display_name = excluded.user_display_name, + user_membership_status = excluded.user_membership_status, + race_id = excluded.race_id, + race_name = excluded.race_name, + race_storage = excluded.race_storage, + lang = excluded.lang, + request_ip = excluded.request_ip, + request_user_agent = excluded.request_user_agent, + return_url = excluded.return_url, + selfie_name = excluded.selfie_name, + selfie_sha256 = excluded.selfie_sha256, + selfie_size_bytes = excluded.selfie_size_bytes, + upload_path = excluded.upload_path, + match_count = excluded.match_count + `); + + const updateSearchOutcomeStatement = db.prepare(` + UPDATE faceai_audit_searches + SET + updated_at = @updated_at, + completed_at = COALESCE(@completed_at, completed_at), + redirect_issued_at = COALESCE(@redirect_issued_at, redirect_issued_at), + status = COALESCE(@status, status), + completion_code = COALESCE(@completion_code, completion_code), + error_code = CASE + WHEN @clear_error = 1 THEN NULL + ELSE COALESCE(@error_code, error_code) + END, + error_message = CASE + WHEN @clear_error = 1 THEN NULL + ELSE COALESCE(@error_message, error_message) + END, + result_id = COALESCE(@result_id, result_id), + match_count = COALESCE(@match_count, match_count), + matches_json = COALESCE(@matches_json, matches_json) + WHERE search_id = @search_id + `); + + const insertEventStatement = db.prepare(` + INSERT INTO faceai_audit_events ( + search_id, + event_type, + happened_at, + status, + user_id, + user_display_name, + race_id, + race_name, + race_storage, + request_ip, + request_user_agent, + selfie_sha256, + result_id, + match_count, + completion_code, + error_code, + payload_json + ) VALUES ( + @search_id, + @event_type, + @happened_at, + @status, + @user_id, + @user_display_name, + @race_id, + @race_name, + @race_storage, + @request_ip, + @request_user_agent, + @selfie_sha256, + @result_id, + @match_count, + @completion_code, + @error_code, + @payload_json + ) + `); + + const pruneSearchesStatement = db.prepare(` + DELETE FROM faceai_audit_searches + WHERE requested_at < @cutoff + `); + + const pruneStandaloneEventsStatement = db.prepare(` + DELETE FROM faceai_audit_events + WHERE search_id IS NULL AND happened_at < @cutoff + `); + + function maybePrune(now = Date.now()) { + if (now - lastPrunedAt < ONE_DAY_MS) { + return; + } + + const cutoff = now - retentionMs; + pruneSearchesStatement.run({ cutoff }); + pruneStandaloneEventsStatement.run({ cutoff }); + lastPrunedAt = now; + } + + function recordEvent({ + eventType, + happenedAt = Date.now(), + searchId = null, + status = null, + user = null, + race = null, + request = null, + selfieFingerprint = null, + resultId = null, + matchCount = null, + completionCode = null, + errorCode = null, + payload = null + }) { + maybePrune(happenedAt); + insertEventStatement.run({ + search_id: textOrNull(searchId), + event_type: eventType, + happened_at: happenedAt, + status: textOrNull(status), + user_id: textOrNull(user?.id), + user_display_name: textOrNull(user?.displayName), + race_id: textOrNull(race?.id), + race_name: textOrNull(race?.name), + race_storage: textOrNull(race?.storage), + request_ip: textOrNull(request?.ip), + request_user_agent: textOrNull(request?.userAgent), + selfie_sha256: textOrNull(selfieFingerprint?.hashHex), + result_id: textOrNull(resultId), + match_count: Number.isFinite(matchCount) ? matchCount : null, + completion_code: textOrNull(completionCode), + error_code: textOrNull(errorCode), + payload_json: jsonOrNull(payload) + }); + } + + function recordSearchRequested({ search, user, race, request, selfieFingerprint, payload = null }) { + const requestedAt = Number(search?.createdAt || Date.now()); + maybePrune(requestedAt); + + upsertSearchRequestStatement.run({ + search_id: String(search.id), + requested_at: requestedAt, + updated_at: requestedAt, + status: textOrNull(search.status) || 'queued', + user_id: String(search.userId), + user_display_name: textOrNull(user?.displayName), + user_membership_status: textOrNull(user?.membershipStatus), + race_id: String(search.raceId), + race_name: textOrNull(race?.name || search.raceName), + race_storage: textOrNull(race?.storage || search.raceStorage), + lang: textOrNull(search.lang), + request_ip: textOrNull(request?.ip), + request_user_agent: textOrNull(request?.userAgent), + return_url: textOrNull(search.returnUrl), + selfie_name: textOrNull(search.selfieName), + selfie_sha256: textOrNull(selfieFingerprint?.hashHex), + selfie_size_bytes: Number.isFinite(selfieFingerprint?.sizeBytes) ? selfieFingerprint.sizeBytes : null, + upload_path: textOrNull(search.uploadPath || search.selfiePath), + match_count: Number(search.matchCount || 0) + }); + + recordEvent({ + eventType: 'search_requested', + happenedAt: requestedAt, + searchId: search.id, + status: search.status || 'queued', + user, + race, + request, + selfieFingerprint, + payload + }); + } + + function markSearchCompleted({ + searchId, + user = null, + race = null, + request = null, + resultId, + matchCount, + matches, + completionCode = null, + completedAt = Date.now(), + payload = null + }) { + maybePrune(completedAt); + + updateSearchOutcomeStatement.run({ + search_id: String(searchId), + updated_at: completedAt, + completed_at: completedAt, + redirect_issued_at: null, + status: 'completed', + completion_code: textOrNull(completionCode), + clear_error: 1, + error_code: null, + error_message: null, + result_id: textOrNull(resultId), + match_count: Number(matchCount || 0), + matches_json: jsonOrNull(matches) + }); + + recordEvent({ + eventType: 'search_completed', + happenedAt: completedAt, + searchId, + status: 'completed', + user, + race, + request, + resultId, + matchCount, + completionCode, + payload + }); + } + + function markSearchFailed({ + searchId, + user = null, + race = null, + request = null, + errorCode, + errorMessage, + completedAt = Date.now(), + payload = null + }) { + maybePrune(completedAt); + + updateSearchOutcomeStatement.run({ + search_id: String(searchId), + updated_at: completedAt, + completed_at: completedAt, + redirect_issued_at: null, + status: 'failed', + completion_code: null, + clear_error: 0, + error_code: textOrNull(errorCode), + error_message: textOrNull(errorMessage), + result_id: null, + match_count: null, + matches_json: null + }); + + recordEvent({ + eventType: 'search_failed', + happenedAt: completedAt, + searchId, + status: 'failed', + user, + race, + request, + errorCode, + payload: { + errorMessage: textOrNull(errorMessage), + ...payload + } + }); + } + + function markRedirectIssued({ + searchId, + user = null, + race = null, + request = null, + resultId, + matchCount, + redirectUrl, + issuedAt = Date.now() + }) { + maybePrune(issuedAt); + + updateSearchOutcomeStatement.run({ + search_id: String(searchId), + updated_at: issuedAt, + completed_at: null, + redirect_issued_at: issuedAt, + status: null, + completion_code: null, + clear_error: 0, + error_code: null, + error_message: null, + result_id: textOrNull(resultId), + match_count: Number(matchCount || 0), + matches_json: null + }); + + recordEvent({ + eventType: 'search_redirect_issued', + happenedAt: issuedAt, + searchId, + status: 'completed', + user, + race, + request, + resultId, + matchCount, + payload: { + redirectUrl: textOrNull(redirectUrl) + } + }); + } + + maybePrune(); + + return { + dbPath, + recordEvent, + recordSearchRequested, + markSearchCompleted, + markSearchFailed, + markRedirectIssued, + close() { + db.close(); + } + }; +} \ No newline at end of file diff --git a/faceai/apps/backend/src/config.js b/faceai/apps/backend/src/config.js index 0dd76373..e799866b 100644 --- a/faceai/apps/backend/src/config.js +++ b/faceai/apps/backend/src/config.js @@ -27,6 +27,8 @@ export const config = { queueName: process.env.FACEAI_QUEUE_NAME || 'faceai-searches', runtimeRoot: process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', uploadRoot: process.env.FACEAI_UPLOAD_ROOT || path.join(process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', 'uploads'), + auditDbPath: process.env.FACEAI_AUDIT_DB_PATH || path.join(process.env.FACEAI_LOG_ROOT || '/data/logs', 'faceai-audit.sqlite'), + auditRetentionDays: Number(process.env.FACEAI_AUDIT_RETENTION_DAYS || 730), searchTtlSeconds: Number(process.env.FACEAI_SEARCH_TTL_SECONDS || 24 * 60 * 60), resultTtlSeconds: Number(process.env.FACEAI_RESULT_TTL_SECONDS || 24 * 60 * 60), processorHeartbeatGraceMs: Number(process.env.FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS || 60 * 1000), diff --git a/faceai/apps/backend/src/server.js b/faceai/apps/backend/src/server.js index 96c729b1..d1a1a90b 100644 --- a/faceai/apps/backend/src/server.js +++ b/faceai/apps/backend/src/server.js @@ -2,11 +2,13 @@ import express from 'express'; import cors from 'cors'; import cookieParser from 'cookie-parser'; import multer from 'multer'; +import crypto from 'node:crypto'; import fs from 'node:fs'; import fsp from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { config } from './config.js'; +import { createAuditStore } from './audit-store.js'; import { signPayload, verifySignedPayload } from './auth.js'; import { createSession, getSession, mockCatalog } from './store.js'; import { buildRaceStorage, resolveRacePklAvailability } from './race-storage.js'; @@ -31,6 +33,7 @@ const frontendDist = path.resolve(__dirname, '../../frontend/dist'); const app = express(); const redis = createRedisConnection(config.redisUrl); const searchQueue = getSearchQueue({ queueName: config.queueName, connection: redis }); +const auditStore = createAuditStore({ dbPath: config.auditDbPath, retentionDays: config.auditRetentionDays }); let lastHealthFailureSignature = null; await fsp.mkdir(config.uploadRoot, { recursive: true }); @@ -85,6 +88,34 @@ function clientIp(req) { return forwardedFor || req.ip || req.socket?.remoteAddress || null; } +function requestAuditContext(req) { + return { + ip: clientIp(req), + userAgent: req.headers['user-agent'] || null + }; +} + +async function fingerprintUpload(filePath) { + const hash = crypto.createHash('sha256'); + let sizeBytes = 0; + + await new Promise((resolve, reject) => { + const stream = fs.createReadStream(filePath); + stream.on('data', (chunk) => { + hash.update(chunk); + sizeBytes += chunk.length; + }); + stream.on('end', resolve); + stream.on('error', reject); + }); + + return { + algorithm: 'sha256', + hashHex: hash.digest('hex'), + sizeBytes + }; +} + function logFaceAiAccess(event, req, details = {}) { console.log(`[FaceAI] ${event} ${JSON.stringify({ ip: clientIp(req), @@ -118,13 +149,34 @@ async function failSearchIfProcessorUnavailable(search) { return search; } - return markSearchFailed( + const failedSearch = await markSearchFailed( redis, search.id, 'PROCESSOR_UNAVAILABLE', processor.message, config.searchTtlSeconds ); + + if (failedSearch) { + auditStore.markSearchFailed({ + searchId: failedSearch.id, + user: { id: failedSearch.userId }, + race: { + id: failedSearch.raceId, + name: failedSearch.raceName, + storage: failedSearch.raceStorage + }, + errorCode: 'PROCESSOR_UNAVAILABLE', + errorMessage: processor.message, + completedAt: failedSearch.completedAt || Date.now(), + payload: { + processorAgeMs: processor.ageMs, + processorHeartbeat: processor.heartbeat + } + }); + } + + return failedSearch; } async function resolveBlockingActiveSearch(userId) { @@ -534,6 +586,18 @@ app.post('/api/searches', requireSession, enforceSearchRateLimit, upload.single( const processor = await getProcessorAvailability(); if (!processor.available) { + auditStore.recordEvent({ + eventType: 'search_blocked_processor_unavailable', + user: summarizeUser(req.faceaiSession.user), + race: summarizeRace(race), + request: requestAuditContext(req), + errorCode: 'PROCESSOR_UNAVAILABLE', + payload: { + processorAgeMs: processor.ageMs, + processorHeartbeat: processor.heartbeat + } + }); + logFaceAiAccess('Identification blocked: processor unavailable', req, { user: summarizeUser(req.faceaiSession.user), race: summarizeRace(race), @@ -596,6 +660,7 @@ app.post('/api/searches', requireSession, enforceSearchRateLimit, upload.single( await fsp.mkdir(finalUploadDir, { recursive: true }); const finalUploadPath = path.join(finalUploadDir, path.basename(req.file.path)); await fsp.rename(req.file.path, finalUploadPath); + const selfieFingerprint = await fingerprintUpload(finalUploadPath); const updatedSearch = await saveSearchRecord(redis, { ...search, @@ -603,6 +668,17 @@ app.post('/api/searches', requireSession, enforceSearchRateLimit, upload.single( uploadPath: finalUploadPath }, config.searchTtlSeconds); + auditStore.recordSearchRequested({ + search: updatedSearch, + user: req.faceaiSession.user, + race, + request: requestAuditContext(req), + selfieFingerprint, + payload: { + availability + } + }); + await searchQueue.add('run-search', { searchId: search.id }, { @@ -690,6 +766,16 @@ app.get('/api/searches/:id/redirect', requireSession, async (req, res) => { redirectUrl }); + auditStore.markRedirectIssued({ + searchId: search.id, + user: req.faceaiSession.user, + race: req.faceaiSession.race, + request: requestAuditContext(req), + resultId: result.id, + matchCount: result.matches?.length || 0, + redirectUrl + }); + res.json({ url: redirectUrl }); diff --git a/faceai/apps/processor/src/config.js b/faceai/apps/processor/src/config.js index 80de646f..7f6d55d9 100644 --- a/faceai/apps/processor/src/config.js +++ b/faceai/apps/processor/src/config.js @@ -26,6 +26,8 @@ export const config = { workerTimeoutMs: Number(process.env.FACEAI_WORKER_TIMEOUT_MS || 5 * 60 * 1000), runtimeRoot: process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', logRoot: process.env.FACEAI_LOG_ROOT || path.join(process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', 'logs'), + auditDbPath: process.env.FACEAI_AUDIT_DB_PATH || path.join(process.env.FACEAI_LOG_ROOT || '/data/logs', 'faceai-audit.sqlite'), + auditRetentionDays: Number(process.env.FACEAI_AUDIT_RETENTION_DAYS || 730), pklRoot: process.env.FACEAI_PKL_ROOT || '/data/pkl', matcherBinary: process.env.FACEAI_MATCHER_BINARY || '/app/bin/face_matcher', matcherTolerance, diff --git a/faceai/apps/processor/src/worker-utils.js b/faceai/apps/processor/src/worker-utils.js index 5cb01fbe..7d850598 100644 --- a/faceai/apps/processor/src/worker-utils.js +++ b/faceai/apps/processor/src/worker-utils.js @@ -32,7 +32,7 @@ export async function runFaceMatcher({ matcherBinary, matcherTolerance, selfiePa ]; if (matcherTolerance !== null && matcherTolerance !== undefined) { - matcherArgs.push('--tollerance', String(matcherTolerance)); + matcherArgs.push('--tolerance', String(matcherTolerance)); } const child = spawn(matcherBinary, matcherArgs, { diff --git a/faceai/apps/processor/src/worker.js b/faceai/apps/processor/src/worker.js index c42ab964..8b6ae69b 100644 --- a/faceai/apps/processor/src/worker.js +++ b/faceai/apps/processor/src/worker.js @@ -3,6 +3,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { Worker } from 'bullmq'; import { config } from './config.js'; +import { createAuditStore } from '../../backend/src/audit-store.js'; import { createRedisConnection, getSearchRecord, @@ -16,6 +17,7 @@ import { import { parseMatcherCsv, resolvePklPath, runFaceMatcher } from './worker-utils.js'; const connection = createRedisConnection(config.redisUrl); +const auditStore = createAuditStore({ dbPath: config.auditDbPath, retentionDays: config.auditRetentionDays }); async function ensureMatcherBinaryAvailable() { try { @@ -93,9 +95,23 @@ async function completeSearch(search, searchId, searchLogPath, matchCount, match completionCode }); - await markSearchCompleted(connection, searchId, result.id, matchCount, config.searchTtlSeconds, { + const completedSearch = await markSearchCompleted(connection, searchId, result.id, matchCount, config.searchTtlSeconds, { completionCode }); + auditStore.markSearchCompleted({ + searchId, + user: { id: search.userId }, + race: { + id: search.raceId, + name: search.raceName, + storage: search.raceStorage + }, + resultId: result.id, + matchCount, + matches, + completionCode, + completedAt: completedSearch?.completedAt || Date.now() + }); await releaseActiveSearchLock(connection, search.userId, searchId); } @@ -178,7 +194,22 @@ async function processJob(job) { message: error.message, stack: error.stack || null }); - await markSearchFailed(connection, searchId, 'PROCESSOR_ERROR', error.message, config.searchTtlSeconds); + const failedSearch = await markSearchFailed(connection, searchId, 'PROCESSOR_ERROR', error.message, config.searchTtlSeconds); + auditStore.markSearchFailed({ + searchId, + user: { id: search.userId }, + race: { + id: search.raceId, + name: search.raceName, + storage: search.raceStorage + }, + errorCode: 'PROCESSOR_ERROR', + errorMessage: error.message, + completedAt: failedSearch?.completedAt || Date.now(), + payload: { + stack: error.stack || null + } + }); await releaseActiveSearchLock(connection, search.userId, searchId); throw error; } diff --git a/faceai/docker-compose.override.yml b/faceai/docker-compose.override.yml index 562f3f84..1946c01f 100644 --- a/faceai/docker-compose.override.yml +++ b/faceai/docker-compose.override.yml @@ -1,6 +1,6 @@ services: faceai: - image: ${FACEAI_CLIENT_DEV_IMAGE:-node:20-alpine} + image: ${FACEAI_CLIENT_DEV_IMAGE:-node:22-trixie-slim} working_dir: /app command: - node @@ -26,6 +26,7 @@ services: FACEAI_RUNTIME_ROOT: ${FACEAI_RUNTIME_ROOT:-/data/runtime} FACEAI_UPLOAD_ROOT: ${FACEAI_UPLOAD_ROOT:-/data/runtime/uploads} FACEAI_LOG_ROOT: ${FACEAI_LOG_ROOT:-/data/logs} + FACEAI_AUDIT_DB_PATH: ${FACEAI_AUDIT_DB_PATH:-/data/runtime/faceai-audit.sqlite} FACEAI_PKL_ROOT: ${FACEAI_PKL_ROOT:-/data/pkl} FACEAI_ENABLE_LOCAL_LEGACY_STATIC: ${FACEAI_ENABLE_LOCAL_LEGACY_STATIC:-1} FACEAI_LOCAL_LEGACY_STATIC_ROOT: ${FACEAI_LOCAL_LEGACY_STATIC_ROOT:-/legacy-www} @@ -58,6 +59,7 @@ services: FACEAI_QUEUE_NAME: ${FACEAI_QUEUE_NAME:-faceai-searches} FACEAI_RUNTIME_ROOT: ${FACEAI_RUNTIME_ROOT:-/data/runtime} FACEAI_LOG_ROOT: ${FACEAI_LOG_ROOT:-/data/logs} + FACEAI_AUDIT_DB_PATH: ${FACEAI_AUDIT_DB_PATH:-/data/runtime/faceai-audit.sqlite} FACEAI_PKL_ROOT: ${FACEAI_PKL_ROOT:-/data/pkl} FACEAI_MATCHER_BINARY: ${FACEAI_MATCHER_BINARY:-/app/bin/face_matcher} FACEAI_MATCHER_TOLERANCE: ${FACEAI_MATCHER_TOLERANCE:-0.5} diff --git a/faceai/docker-compose.yml b/faceai/docker-compose.yml index b9c4ff7b..47bc5287 100644 --- a/faceai/docker-compose.yml +++ b/faceai/docker-compose.yml @@ -24,6 +24,8 @@ services: FACEAI_RUNTIME_ROOT: ${FACEAI_RUNTIME_ROOT:-/data/runtime} FACEAI_UPLOAD_ROOT: ${FACEAI_UPLOAD_ROOT:-/data/runtime/uploads} FACEAI_LOG_ROOT: ${FACEAI_LOG_ROOT:-/data/logs} + FACEAI_AUDIT_DB_PATH: ${FACEAI_AUDIT_DB_PATH:-/data/logs/faceai-audit.sqlite} + FACEAI_AUDIT_RETENTION_DAYS: ${FACEAI_AUDIT_RETENTION_DAYS:-730} FACEAI_PKL_ROOT: ${FACEAI_PKL_ROOT:-/data/pkl} FACEAI_ENABLE_LOCAL_LEGACY_STATIC: ${FACEAI_ENABLE_LOCAL_LEGACY_STATIC:-0} volumes: @@ -59,6 +61,8 @@ services: FACEAI_QUEUE_NAME: ${FACEAI_QUEUE_NAME:-faceai-searches} FACEAI_RUNTIME_ROOT: ${FACEAI_RUNTIME_ROOT:-/data/runtime} FACEAI_LOG_ROOT: ${FACEAI_LOG_ROOT:-/data/logs} + FACEAI_AUDIT_DB_PATH: ${FACEAI_AUDIT_DB_PATH:-/data/logs/faceai-audit.sqlite} + FACEAI_AUDIT_RETENTION_DAYS: ${FACEAI_AUDIT_RETENTION_DAYS:-730} FACEAI_PKL_ROOT: ${FACEAI_PKL_ROOT:-/data/pkl} FACEAI_MATCHER_BINARY: ${FACEAI_MATCHER_BINARY:-/app/bin/face_matcher} FACEAI_MATCHER_TOLERANCE: ${FACEAI_MATCHER_TOLERANCE:-0.5} diff --git a/faceai/tests/e2e/faceai-simulator.spec.js b/faceai/tests/e2e/faceai-simulator.spec.js index 6610ec47..8f791b3c 100644 --- a/faceai/tests/e2e/faceai-simulator.spec.js +++ b/faceai/tests/e2e/faceai-simulator.spec.js @@ -3,26 +3,35 @@ const { ensureLocalAuthenticatedRacePage, EXPECTED_MATCH_COUNT, FACEAI_BASE_URL, + LEGACY_BASE_URL, LEGACY_RACE_ID, SELFIE_NAME, buildHandoffUrl, buildSimulatorUrl, getSearchArtifacts, getSelfiePath, + readAuditArtifacts, readUtf8 } = require('./faceai-test-utils'); const FACEAI_HOME_URL_RE = /http:\/\/(localhost|127\.0\.0\.1):3001\/?(?:\?.*)?$/; const FACEAI_CALLBACK_URL_RE = /http:\/\/(localhost|127\.0\.0\.1):3001\/auth\/callback\?token=/; -const LEGACY_HOME_URL_RE = /http:\/\/(localhost|127\.0\.0\.1):8080\/index\.jsp$/; const LONG_TEST_TIMEOUT_MS = 3 * 60 * 1000; const SHORT_UI_TIMEOUT_MS = 30 * 1000; const SEARCH_COMPLETION_TIMEOUT_MS = 75 * 1000; const LEGACY_RETURN_TIMEOUT_MS = 75 * 1000; const FILE_CHOOSER_TIMEOUT_MS = 8 * 1000; +const FACEAI_CONSENT_HEADING_RE = /Prima di continuare|Before you continue/i; +const FACEAI_UPLOAD_HEADING_RE = /Carica il tuo selfie|Upload your selfie/i; +const LEGACY_BASE_URL_RE = new RegExp(`^${escapeRegExp(LEGACY_BASE_URL)}`); +const LEGACY_HOME_URL_RE = new RegExp(`^${escapeRegExp(`${LEGACY_BASE_URL}/index.jsp`)}$`); + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} function buildLegacySimulatorReturnMatcher(raceId) { - return new RegExp(`http://(localhost|127\\.0\\.0\\.1):8080/Foto2\\.abl\\?id_gara=${raceId}.*`); + return new RegExp(`^${escapeRegExp(`${LEGACY_BASE_URL}/Foto2.abl?id_gara=${raceId}`)}.*`); } function assertLogDoesNotContain(content, patterns, label) { @@ -32,10 +41,20 @@ function assertLogDoesNotContain(content, patterns, label) { } async function waitForFaceAiHome(page) { - await page.waitForURL((url) => FACEAI_CALLBACK_URL_RE.test(url.toString()) || FACEAI_HOME_URL_RE.test(url.toString()), { + await page.waitForURL((url) => FACEAI_HOME_URL_RE.test(url.toString()), { timeout: SHORT_UI_TIMEOUT_MS }); - await expect(page.getByRole('heading', { name: 'Trova le tue foto con un selfie' })).toBeVisible(); + + const consentHeading = page.getByRole('heading', { name: FACEAI_CONSENT_HEADING_RE }); + if (await consentHeading.isVisible().catch(() => false)) { + await page.getByRole('checkbox', { + name: /Confermo di aver letto l’informativa sul trattamento dei dati biometrici\.|I confirm that I have read the biometric data processing notice\./i + }).check(); + await page.getByRole('button', { name: /Accetto e continuo|I agree and continue/i }).click(); + } + + await expect(page.getByRole('heading', { name: FACEAI_UPLOAD_HEADING_RE })).toBeVisible(); + await expect(page.getByRole('button', { name: /Scegli immagine|Choose image/i })).toBeVisible(); } async function launchFromSimulator(page, options = {}) { @@ -59,11 +78,10 @@ async function readLaunchUrlFromLegacyPage(page) { }); expect(launchUrl, 'Expected the legacy race page to expose a FaceAI handoff URL builder.').toBeTruthy(); - return new URL(launchUrl, 'http://127.0.0.1:8080'); + return new URL(launchUrl, LEGACY_BASE_URL); } async function startSearch(page, selfieName) { - const selfieLabel = selfieName.split(/[\\/]+/u).pop(); const createResponsePromise = page.waitForResponse((response) => { return response.url().includes('/api/searches') && response.request().method() === 'POST' @@ -71,8 +89,6 @@ async function startSearch(page, selfieName) { }); await page.locator('input[type="file"]').setInputFiles(getSelfiePath(selfieName)); - await expect(page.getByText(selfieLabel)).toBeVisible(); - await page.getByRole('button', { name: 'Avvia ricerca Face ID' }).click(); const createResponse = await createResponsePromise; return createResponse.json(); @@ -151,6 +167,79 @@ async function verifySearchLogs(searchId, { expectedMatchCount, expectedSelfieNa return { backendLog, processorLog, workerLog, matcherLog }; } +async function verifyAuditLog(searchId, { expectedMatchCount, expectedRaceId, expectedSelfieName, expectedUserId }) { + const artifacts = getSearchArtifacts(searchId); + const audit = readAuditArtifacts(searchId); + + expect(audit.searchRow, `Expected ${artifacts.auditDbPath} to contain an audit row for ${searchId}`).toBeTruthy(); + expect(audit.searchRow.search_id).toBe(searchId); + expect(audit.searchRow.status).toBe('completed'); + expect(audit.searchRow.match_count).toBe(expectedMatchCount); + expect(audit.searchRow.race_id).toBe(expectedRaceId); + expect(audit.searchRow.user_id).toBe(expectedUserId); + expect(audit.searchRow.selfie_name).toBe(expectedSelfieName); + expect(audit.searchRow.selfie_sha256).toMatch(/^[a-f0-9]{64}$/i); + expect(audit.searchRow.selfie_size_bytes).toBeGreaterThan(0); + expect(audit.searchRow.result_id).toBeTruthy(); + expect(audit.searchRow.requested_at).toBeGreaterThan(0); + expect(audit.searchRow.completed_at).toBeGreaterThan(0); + expect(audit.searchRow.redirect_issued_at).toBeGreaterThan(0); + expect(audit.searchRow.matches).toHaveLength(expectedMatchCount); + + const eventTypes = audit.events.map((event) => event.event_type); + expect(eventTypes).toContain('search_requested'); + expect(eventTypes).toContain('search_completed'); + expect(eventTypes).toContain('search_redirect_issued'); + + const requestedEvent = audit.events.find((event) => event.event_type === 'search_requested'); + expect(requestedEvent?.selfie_sha256).toBe(audit.searchRow.selfie_sha256); + + const fingerprintMatch = audit.fingerprintMatches.find((entry) => entry.search_id === searchId); + expect(fingerprintMatch, 'Expected fingerprint lookup to find the original search row').toBeTruthy(); + expect(fingerprintMatch.match_count).toBe(expectedMatchCount); + expect(fingerprintMatch.result_id).toBe(audit.searchRow.result_id); + expect(fingerprintMatch.status).toBe('completed'); + + return audit; +} + +async function verifyNoResultsAuditLog(searchId, { expectedRaceId, expectedSelfieName, expectedUserId }) { + const artifacts = getSearchArtifacts(searchId); + const audit = readAuditArtifacts(searchId); + + expect(audit.searchRow, `Expected ${artifacts.auditDbPath} to contain an audit row for ${searchId}`).toBeTruthy(); + expect(audit.searchRow.search_id).toBe(searchId); + expect(audit.searchRow.status).toBe('completed'); + expect(audit.searchRow.completion_code).toBe('NO_FACES_FOUND'); + expect(audit.searchRow.match_count).toBe(0); + expect(audit.searchRow.race_id).toBe(expectedRaceId); + expect(audit.searchRow.user_id).toBe(expectedUserId); + expect(audit.searchRow.selfie_name).toBe(expectedSelfieName); + expect(audit.searchRow.selfie_sha256).toMatch(/^[a-f0-9]{64}$/i); + expect(audit.searchRow.selfie_size_bytes).toBeGreaterThan(0); + expect(audit.searchRow.result_id).toBeTruthy(); + expect(audit.searchRow.requested_at).toBeGreaterThan(0); + expect(audit.searchRow.completed_at).toBeGreaterThan(0); + expect(audit.searchRow.redirect_issued_at).toBeNull(); + expect(audit.searchRow.matches).toHaveLength(0); + + const eventTypes = audit.events.map((event) => event.event_type); + expect(eventTypes).toContain('search_requested'); + expect(eventTypes).toContain('search_completed'); + expect(eventTypes).not.toContain('search_redirect_issued'); + + const requestedEvent = audit.events.find((event) => event.event_type === 'search_requested'); + expect(requestedEvent?.selfie_sha256).toBe(audit.searchRow.selfie_sha256); + + const fingerprintMatch = audit.fingerprintMatches.find((entry) => entry.search_id === searchId); + expect(fingerprintMatch, 'Expected fingerprint lookup to find the original search row').toBeTruthy(); + expect(fingerprintMatch.match_count).toBe(0); + expect(fingerprintMatch.result_id).toBe(audit.searchRow.result_id); + expect(fingerprintMatch.status).toBe('completed'); + + return audit; +} + async function closeContexts(contexts) { await Promise.all(contexts.map(async (context) => { try { @@ -181,6 +270,41 @@ test('runs the legacy Tomcat flow through FaceAI and returns to the filtered leg expectedMatchCount: EXPECTED_MATCH_COUNT, expectedSelfieName: SELFIE_NAME.split(/[\\/]+/u).pop() }); + + await verifyAuditLog(search.id, { + expectedMatchCount: EXPECTED_MATCH_COUNT, + expectedRaceId: LEGACY_RACE_ID, + expectedSelfieName: SELFIE_NAME.split(/[\\/]+/u).pop(), + expectedUserId: 'legacy-user-1' + }); +}); + +test('records structured logs for a completed no-results FaceAI search in the dev compose stack', async ({ page }) => { + test.slow(); + + await launchFromSimulator(page, { + raceId: LEGACY_RACE_ID, + raceSlug: 'isolotto', + raceName: 'Festa sociale UP Isolotto', + raceFolder: 'ISOLOTTO' + }); + + const search = await startSearch(page, SELFIE_NAME); + + await waitForSearchCondition(page, search.id, (payload) => { + return payload.status === 'completed' && payload.completionCode === 'NO_FACES_FOUND'; + }, SEARCH_COMPLETION_TIMEOUT_MS); + + await verifySearchLogs(search.id, { + expectedMatchCount: 0, + expectedSelfieName: SELFIE_NAME.split(/[\\/]+/u).pop() + }); + + await verifyNoResultsAuditLog(search.id, { + expectedRaceId: LEGACY_RACE_ID, + expectedSelfieName: SELFIE_NAME.split(/[\\/]+/u).pop(), + expectedUserId: 'legacy-user-1' + }); }); test('builds the legacy FaceAI handoff URL with the exact local race storage metadata', async ({ page }) => { diff --git a/faceai/tests/e2e/faceai-test-utils.js b/faceai/tests/e2e/faceai-test-utils.js index e5e9d5b7..ed535ad3 100644 --- a/faceai/tests/e2e/faceai-test-utils.js +++ b/faceai/tests/e2e/faceai-test-utils.js @@ -1,11 +1,17 @@ const fs = require('node:fs/promises'); +const fsSync = require('node:fs'); const path = require('node:path'); -const { spawn } = require('node:child_process'); +const { spawn, spawnSync } = require('node:child_process'); +const { DatabaseSync } = require('node:sqlite'); const ROOT_DIR = path.resolve(__dirname, '..', '..'); const WORKSPACE_ROOT = path.resolve(ROOT_DIR, '..'); const LOG_ROOT = path.join(ROOT_DIR, 'logs'); const SEARCH_LOG_ROOT = path.join(LOG_ROOT, 'searches'); +const AUDIT_DB_PATH = path.join(LOG_ROOT, 'faceai-audit.sqlite'); +const PREFER_CONTAINER_AUDIT_DB = process.env.FACEAI_E2E_AUDIT_READ_FROM_CONTAINER === '1' || process.platform === 'win32'; +const AUDIT_DB_PATH_IN_CONTAINER = process.env.FACEAI_E2E_AUDIT_DB_PATH_IN_CONTAINER || '/data/runtime/faceai-audit.sqlite'; +const AUDIT_DB_QUERY_CONTAINER = process.env.FACEAI_E2E_AUDIT_QUERY_CONTAINER || 'regalami-faceai-processor'; const FACEAI_BASE_URL = process.env.FACEAI_E2E_BASE_URL || 'http://127.0.0.1:3001'; const SIMULATOR_URL = process.env.FACEAI_E2E_SIMULATOR_URL || 'http://127.0.0.1:8080/Foto2.abl?id_gara=1018547&pageRow=96&pageNumber=1'; const LEGACY_BASE_URL = process.env.FACEAI_E2E_LEGACY_BASE_URL || 'http://127.0.0.1:8080'; @@ -260,11 +266,182 @@ function getSearchArtifacts(searchId) { searchRoot, backendLogPath: path.join(LOG_ROOT, 'backend.log'), processorLogPath: path.join(LOG_ROOT, 'processor.log'), + auditDbPath: fsSync.existsSync(AUDIT_DB_PATH) ? AUDIT_DB_PATH : `${AUDIT_DB_QUERY_CONTAINER}:${AUDIT_DB_PATH_IN_CONTAINER}`, workerLogPath: path.join(searchRoot, 'worker.log'), matcherLogPath: path.join(searchRoot, 'matcher.log') }; } +function parseAuditArtifacts(result) { + return { + searchRow: result.searchRow ? { + ...result.searchRow, + matches: result.searchRow.matches_json ? JSON.parse(result.searchRow.matches_json) : null + } : null, + events: result.events.map((event) => ({ + ...event, + payload: event.payload_json ? JSON.parse(event.payload_json) : null + })), + fingerprintMatches: result.fingerprintMatches || [] + }; +} + +function readAuditArtifactsFromContainer(searchId) { + const script = ` + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(${JSON.stringify(AUDIT_DB_PATH_IN_CONTAINER)}, { readOnly: true }); + + try { + const searchId = ${JSON.stringify(String(searchId))}; + const searchRow = db.prepare(\` + SELECT + search_id, + requested_at, + completed_at, + redirect_issued_at, + status, + completion_code, + error_code, + error_message, + user_id, + user_display_name, + user_membership_status, + race_id, + race_name, + race_storage, + lang, + request_ip, + request_user_agent, + return_url, + selfie_name, + selfie_sha256, + selfie_size_bytes, + upload_path, + result_id, + match_count, + matches_json + FROM faceai_audit_searches + WHERE search_id = ? + \`).get(searchId); + + const events = db.prepare(\` + SELECT + event_type, + happened_at, + status, + user_id, + race_id, + selfie_sha256, + result_id, + match_count, + completion_code, + error_code, + payload_json + FROM faceai_audit_events + WHERE search_id = ? + ORDER BY happened_at ASC, id ASC + \`).all(searchId); + + const fingerprintMatches = searchRow?.selfie_sha256 + ? db.prepare(\` + SELECT search_id, result_id, match_count, status + FROM faceai_audit_searches + WHERE selfie_sha256 = ? + ORDER BY requested_at DESC + \`).all(searchRow.selfie_sha256) + : []; + + console.log(JSON.stringify({ searchRow, events, fingerprintMatches })); + } finally { + db.close(); + } + `; + + const result = spawnSync('docker', ['exec', AUDIT_DB_QUERY_CONTAINER, 'node', '-e', script], { + cwd: ROOT_DIR, + encoding: 'utf8' + }); + + if (result.status !== 0) { + const details = (result.stderr || result.stdout || '').trim(); + throw new Error(`Failed to read audit DB from ${AUDIT_DB_QUERY_CONTAINER}:${AUDIT_DB_PATH_IN_CONTAINER}${details ? `\n${details}` : ''}`); + } + + return parseAuditArtifacts(JSON.parse(result.stdout)); +} + +function readAuditArtifacts(searchId) { + if (PREFER_CONTAINER_AUDIT_DB || !fsSync.existsSync(AUDIT_DB_PATH)) { + return readAuditArtifactsFromContainer(searchId); + } + + const db = new DatabaseSync(AUDIT_DB_PATH, { readOnly: true }); + + try { + const searchRow = db.prepare(` + SELECT + search_id, + requested_at, + completed_at, + redirect_issued_at, + status, + completion_code, + error_code, + error_message, + user_id, + user_display_name, + user_membership_status, + race_id, + race_name, + race_storage, + lang, + request_ip, + request_user_agent, + return_url, + selfie_name, + selfie_sha256, + selfie_size_bytes, + upload_path, + result_id, + match_count, + matches_json + FROM faceai_audit_searches + WHERE search_id = ? + `).get(String(searchId)); + + const events = db.prepare(` + SELECT + event_type, + happened_at, + status, + user_id, + race_id, + selfie_sha256, + result_id, + match_count, + completion_code, + error_code, + payload_json + FROM faceai_audit_events + WHERE search_id = ? + ORDER BY happened_at ASC, id ASC + `).all(String(searchId)); + + const fingerprintMatches = searchRow?.selfie_sha256 + ? db.prepare(` + SELECT search_id, result_id, match_count, status + FROM faceai_audit_searches + WHERE selfie_sha256 = ? + ORDER BY requested_at DESC + `).all(searchRow.selfie_sha256) + : []; + + return parseAuditArtifacts({ searchRow, events, fingerprintMatches }); + } finally { + db.close(); + } +} + async function readUtf8(filePath) { return fs.readFile(filePath, 'utf8'); } @@ -273,6 +450,7 @@ module.exports = { ROOT_DIR, LOG_ROOT, SEARCH_LOG_ROOT, + AUDIT_DB_PATH, FACEAI_BASE_URL, LEGACY_BASE_URL, LEGACY_HOME_URL, @@ -290,6 +468,7 @@ module.exports = { ensureLocalAuthenticatedRacePage, expectLocalRacePageLoaded, getSearchArtifacts, + readAuditArtifacts, getSelfiePath, performLocalLoginRequest, prepareHostState, diff --git a/faceai/tmp/faceai-no-results-smoke.json b/faceai/tmp/faceai-no-results-smoke.json new file mode 100644 index 00000000..dc402876 --- /dev/null +++ b/faceai/tmp/faceai-no-results-smoke.json @@ -0,0 +1,128 @@ +{ + "config": { + "configFile": "K:\\various\\regalamiunsorriso\\faceai\\playwright.config.js", + "rootDir": "K:/various/regalamiunsorriso/faceai/tests/e2e", + "forbidOnly": false, + "fullyParallel": false, + "globalSetup": "K:\\various\\regalamiunsorriso\\faceai\\tests\\e2e\\global-setup.js", + "globalTeardown": "K:\\various\\regalamiunsorriso\\faceai\\tests\\e2e\\global-teardown.js", + "globalTimeout": 0, + "grep": {}, + "grepInvert": null, + "maxFailures": 0, + "metadata": { + "actualWorkers": 1 + }, + "preserveOutput": "always", + "projects": [ + { + "outputDir": "K:/various/regalamiunsorriso/faceai/test-results", + "repeatEach": 1, + "retries": 0, + "metadata": { + "actualWorkers": 1 + }, + "id": "", + "name": "", + "testDir": "K:/various/regalamiunsorriso/faceai/tests/e2e", + "testIgnore": [], + "testMatch": [ + "**/*.@(spec|test).?(c|m)[jt]s?(x)" + ], + "timeout": 60000 + } + ], + "quiet": false, + "reporter": [ + [ + "json" + ] + ], + "reportSlowTests": { + "max": 5, + "threshold": 300000 + }, + "shard": null, + "tags": [], + "updateSnapshots": "missing", + "updateSourceMethod": "patch", + "version": "1.59.1", + "workers": 1, + "webServer": null + }, + "suites": [ + { + "title": "faceai-simulator.spec.js", + "file": "faceai-simulator.spec.js", + "column": 0, + "line": 0, + "specs": [ + { + "title": "records structured logs for a completed no-results FaceAI search in the dev compose stack", + "ok": true, + "tags": [], + "tests": [ + { + "timeout": 180000, + "annotations": [ + { + "type": "slow", + "location": { + "file": "K:\\various\\regalamiunsorriso\\faceai\\tests\\e2e\\faceai-simulator.spec.js", + "line": 283, + "column": 8 + } + } + ], + "expectedStatus": "passed", + "projectId": "", + "projectName": "", + "results": [ + { + "workerIndex": 0, + "parallelIndex": 0, + "status": "passed", + "duration": 4768, + "errors": [], + "stdout": [], + "stderr": [ + { + "text": "(node:36288) ExperimentalWarning: SQLite is an experimental feature and might change at any time\n(Use `node --trace-warnings ...` to show where the warning was created)\n" + } + ], + "retry": 0, + "startTime": "2026-05-19T21:25:46.138Z", + "annotations": [ + { + "type": "slow", + "location": { + "file": "K:\\various\\regalamiunsorriso\\faceai\\tests\\e2e\\faceai-simulator.spec.js", + "line": 283, + "column": 8 + } + } + ], + "attachments": [] + } + ], + "status": "expected" + } + ], + "id": "3529663bd1948fd400e2-8c368f494576987888aa", + "file": "faceai-simulator.spec.js", + "line": 282, + "column": 1 + } + ] + } + ], + "errors": [], + "stats": { + "startTime": "2026-05-19T21:24:59.845Z", + "duration": 51905.866, + "expected": 1, + "skipped": 0, + "unexpected": 0, + "flaky": 0 + } +} diff --git a/stacks/faceai.yml b/stacks/faceai.yml index aa38b676..41edb786 100644 --- a/stacks/faceai.yml +++ b/stacks/faceai.yml @@ -24,6 +24,8 @@ services: FACEAI_RUNTIME_ROOT: /data/runtime FACEAI_UPLOAD_ROOT: /data/runtime/uploads FACEAI_LOG_ROOT: /data/logs + FACEAI_AUDIT_DB_PATH: /data/logs/faceai-audit.sqlite + FACEAI_AUDIT_RETENTION_DAYS: 730 FACEAI_PKL_ROOT: /data/pkl FACEAI_ENABLE_LOCAL_LEGACY_STATIC: 0 volumes: @@ -59,6 +61,8 @@ services: FACEAI_QUEUE_NAME: faceai-searches FACEAI_RUNTIME_ROOT: /data/runtime FACEAI_LOG_ROOT: /data/logs + FACEAI_AUDIT_DB_PATH: /data/logs/faceai-audit.sqlite + FACEAI_AUDIT_RETENTION_DAYS: 730 FACEAI_PKL_ROOT: /data/pkl FACEAI_MATCHER_BINARY: /app/bin/face_matcher FACEAI_MATCHER_TOLERANCE: 0.5