feat(logging): enhance debug logging and cleanup capabilities with new configuration options

faceai/apps/processor/src/config.js

@@ -25,9 +25,12 @@ export const config = {
   workerConcurrency: Number(process.env.FACEAI_WORKER_CONCURRENCY || 2),
   workerTimeoutMs: Number(process.env.FACEAI_WORKER_TIMEOUT_MS || 5 * 60 * 1000),
   runtimeRoot: process.env.FACEAI_RUNTIME_ROOT || '/data/runtime',
+  uploadRoot: process.env.FACEAI_UPLOAD_ROOT || path.join(process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', 'uploads'),
   logRoot: process.env.FACEAI_LOG_ROOT || path.join(process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', 'logs'),
   auditDbPath: process.env.FACEAI_AUDIT_DB_PATH || path.join(process.env.FACEAI_LOG_ROOT || '/data/logs', 'faceai-audit.sqlite'),
   auditRetentionDays: Number(process.env.FACEAI_AUDIT_RETENTION_DAYS || 730),
+  debugRetentionDays: Number(process.env.FACEAI_DEBUG_RETENTION_DAYS || 3),
+  debugCleanupIntervalMs: Number(process.env.FACEAI_DEBUG_CLEANUP_INTERVAL_MS || 6 * 60 * 60 * 1000),
   pklRoot: process.env.FACEAI_PKL_ROOT || '/data/pkl',
   matcherBinary: process.env.FACEAI_MATCHER_BINARY || '/app/bin/face_matcher',
   matcherTolerance,

faceai/apps/processor/src/worker.js

@@ -18,6 +18,7 @@ import { parseMatcherCsv, resolvePklPath, runFaceMatcher } from './worker-utils.
 
 const connection = createRedisConnection(config.redisUrl);
 const auditStore = createAuditStore({ dbPath: config.auditDbPath, retentionDays: config.auditRetentionDays });
+const ONE_DAY_MS = 24 * 60 * 60 * 1000;
 
 async function ensureMatcherBinaryAvailable() {
   try {
@@ -66,6 +67,88 @@ async function appendSearchLog(logPath, message, details) {
   await fs.appendFile(logPath, formatLogLine(message, details), 'utf8');
 }
 
+async function removeDirectoryIfPresent(dirPath) {
+  if (!dirPath) {
+    return;
+  }
+
+  await fs.rm(dirPath, { recursive: true, force: true });
+}
+
+async function cleanupSearchArtifacts(search, searchDir) {
+  const cleanupTargets = new Set([searchDir]);
+
+  if (search?.selfiePath) {
+    cleanupTargets.add(path.dirname(search.selfiePath));
+  }
+
+  await Promise.all(Array.from(cleanupTargets, (target) => removeDirectoryIfPresent(target)));
+}
+
+async function listDirectories(rootPath) {
+  try {
+    const entries = await fs.readdir(rootPath, { withFileTypes: true });
+    return entries.filter((entry) => entry.isDirectory());
+  } catch (error) {
+    if (error?.code === 'ENOENT') {
+      return [];
+    }
+
+    throw error;
+  }
+}
+
+async function pruneExpiredDirectories(rootPath, cutoffTime) {
+  const directories = await listDirectories(rootPath);
+  const removed = [];
+
+  await Promise.all(directories.map(async (entry) => {
+    const targetPath = path.join(rootPath, entry.name);
+    const stats = await fs.stat(targetPath).catch((error) => {
+      if (error?.code === 'ENOENT') {
+        return null;
+      }
+
+      throw error;
+    });
+
+    if (!stats || stats.mtimeMs >= cutoffTime) {
+      return;
+    }
+
+    await removeDirectoryIfPresent(targetPath);
+    removed.push(entry.name);
+  }));
+
+  return removed;
+}
+
+async function pruneDebugArtifacts(now = Date.now()) {
+  const retentionMs = Math.max(1, config.debugRetentionDays) * ONE_DAY_MS;
+  const cutoffTime = now - retentionMs;
+
+  const [uploadDirs, runtimeDirs, logDirs] = await Promise.all([
+    pruneExpiredDirectories(config.uploadRoot, cutoffTime),
+    pruneExpiredDirectories(path.join(config.runtimeRoot, 'searches'), cutoffTime),
+    pruneExpiredDirectories(path.join(config.logRoot, 'searches'), cutoffTime)
+  ]);
+
+  return { uploadDirs, runtimeDirs, logDirs };
+}
+
+async function runScheduledDebugCleanup() {
+  try {
+    const removed = await pruneDebugArtifacts();
+    const removedCount = removed.uploadDirs.length + removed.runtimeDirs.length + removed.logDirs.length;
+
+    if (removedCount > 0) {
+      console.log(`Pruned FaceAI debug artifacts: uploads=${removed.uploadDirs.length}, runtime=${removed.runtimeDirs.length}, logs=${removed.logDirs.length}`);
+    }
+  } catch (error) {
+    console.error('Unable to prune FaceAI debug artifacts:', error);
+  }
+}
+
 async function resolveCompletionCode(logPath, matchCount) {
   if (matchCount > 0) {
     return null;
@@ -212,11 +295,14 @@ async function processJob(job) {
     });
     await releaseActiveSearchLock(connection, search.userId, searchId);
     throw error;
+  } finally {
+    await cleanupSearchArtifacts(search, searchDir);
   }
 }
 
 await ensureMatcherBinaryAvailable();
 await publishProcessorHeartbeat();
+await runScheduledDebugCleanup();
 
 const heartbeatTimer = setInterval(() => {
   publishProcessorHeartbeat();
@@ -224,6 +310,12 @@ const heartbeatTimer = setInterval(() => {
 
 heartbeatTimer.unref();
 
+const debugCleanupTimer = setInterval(() => {
+  runScheduledDebugCleanup();
+}, config.debugCleanupIntervalMs);
+
+debugCleanupTimer.unref();
+
 const worker = new Worker(config.queueName, processJob, {
   connection,
   concurrency: config.workerConcurrency