Add processor heartbeat management and improve health check functionality

- Introduced processor heartbeat configuration in environment variables and Docker setup.
- Implemented heartbeat publishing in the processor worker.
- Enhanced health check endpoint to include processor availability status.
- Updated frontend components to handle processor unavailability messages.
- Added legacy return functionality in the upload panel.

faceai/.env.example

@@ -14,6 +14,9 @@ FACEAI_UPLOAD_ROOT=/data/runtime/uploads
 FACEAI_LOG_ROOT=/data/logs
 FACEAI_PKL_ROOT=/data/pkl
 FACEAI_MATCHER_BINARY=/app/bin/face_matcher
+FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS=20000
+FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS=5000
+FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS=20
 LIVE_SITE_BASE_URL=https://www.regalamiunsorriso.it
 LIVE_SITE_LOGIN_URL=https://www.regalamiunsorriso.it/login_clienti-it.html
 LIVE_SITE_RACE_URL=https://www.regalamiunsorriso.it/42%20HALF%20MARATHON%20FIRENZE_gara-1018545---96-1.html

faceai/README.md

@@ -291,6 +291,7 @@ services:
       FACEAI_UPLOAD_ROOT: /data/runtime/uploads
       FACEAI_LOG_ROOT: /data/logs
       FACEAI_PKL_ROOT: /data/pkl
+      FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS: 20000
       FACEAI_ENABLE_LOCAL_LEGACY_STATIC: 0
     volumes:
       - /mnt/storage/data/faceai/runtime:/data/runtime
@@ -299,7 +300,7 @@ services:
     ports:
       - "3001:3001"
     healthcheck:
-      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/health | grep -q '\"ok\":true'"]
+      test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3001/health').then(async (response) => { const payload = await response.json().catch(() => ({})); if (!response.ok || payload.ok !== true) { console.error(JSON.stringify(payload)); process.exit(1); } }).catch((error) => { console.error(error.stack || error.message); process.exit(1); })"]
       interval: 10s
       timeout: 5s
       retries: 6
@@ -326,7 +327,9 @@ services:
       FACEAI_RUNTIME_ROOT: /data/runtime
       FACEAI_LOG_ROOT: /data/logs
       FACEAI_PKL_ROOT: /data/pkl
-      FACEAI_MATCHER_BINARY: /opt/face-recognition/face_matcher
+      FACEAI_MATCHER_BINARY: /app/bin/face_matcher
+      FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS: 5000
+      FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS: 20
       FACEAI_WORKER_CONCURRENCY: 2
       FACEAI_WORKER_TIMEOUT_MS: 300000
     volumes:

faceai/apps/backend/src/config.js

@@ -28,6 +28,7 @@ export const config = {
   uploadRoot: process.env.FACEAI_UPLOAD_ROOT || path.join(process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', 'uploads'),
   searchTtlSeconds: Number(process.env.FACEAI_SEARCH_TTL_SECONDS || 24 * 60 * 60),
   resultTtlSeconds: Number(process.env.FACEAI_RESULT_TTL_SECONDS || 24 * 60 * 60),
+  processorHeartbeatGraceMs: Number(process.env.FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS || 20 * 1000),
   rateLimitWindowSeconds: Number(process.env.FACEAI_RATE_LIMIT_WINDOW_SECONDS || 10 * 60),
   rateLimitMaxRequests: Number(process.env.FACEAI_RATE_LIMIT_MAX_REQUESTS || 5)
 };

faceai/apps/backend/src/redis-store.js

@@ -24,6 +24,10 @@ function rateLimitKey(userId) {
   return `faceai:rate-limit:${userId}`;
 }
 
+function processorHeartbeatKey() {
+  return 'faceai:processor-heartbeat';
+}
+
 export async function incrementRateLimit(redis, userId, windowSeconds) {
   const key = rateLimitKey(userId);
   const count = await redis.incr(key);
@@ -50,6 +54,21 @@ export async function getActiveSearchId(redis, userId) {
   return redis.get(activeSearchKey(userId));
 }
 
+export async function updateProcessorHeartbeat(redis, ttlSeconds, payload = {}) {
+  const heartbeat = {
+    updatedAt: Date.now(),
+    ...payload
+  };
+
+  await redis.set(processorHeartbeatKey(), JSON.stringify(heartbeat), 'EX', ttlSeconds);
+  return heartbeat;
+}
+
+export async function getProcessorHeartbeat(redis) {
+  const raw = await redis.get(processorHeartbeatKey());
+  return raw ? JSON.parse(raw) : null;
+}
+
 export async function createSearchRecord(redis, payload, ttlSeconds) {
   const searchId = randomId('search');
   const record = {

faceai/apps/backend/src/server.js

@@ -15,9 +15,11 @@ import {
   createRedisConnection,
   createSearchRecord,
   getActiveSearchId,
+  getProcessorHeartbeat,
   getResultRecord,
   getSearchRecord,
   incrementRateLimit,
+  markSearchFailed,
   saveSearchRecord
 } from './redis-store.js';
 import { getSearchQueue } from './queue.js';
@@ -28,6 +30,7 @@ const frontendDist = path.resolve(__dirname, '../../frontend/dist');
 const app = express();
 const redis = createRedisConnection(config.redisUrl);
 const searchQueue = getSearchQueue({ queueName: config.queueName, connection: redis });
+let lastHealthFailureSignature = null;
 
 await fsp.mkdir(config.uploadRoot, { recursive: true });
 
@@ -89,6 +92,98 @@ function logFaceAiAccess(event, req, details = {}) {
   })}`);
 }
 
+async function getProcessorAvailability() {
+  const heartbeat = await getProcessorHeartbeat(redis);
+  const ageMs = heartbeat ? Date.now() - Number(heartbeat.updatedAt || 0) : null;
+  const available = Boolean(heartbeat) && Number.isFinite(ageMs) && ageMs <= config.processorHeartbeatGraceMs;
+
+  return {
+    available,
+    ageMs,
+    heartbeat,
+    message: available
+      ? null
+      : 'FaceAI processor is temporarily unavailable. Please try again shortly.'
+  };
+}
+
+async function failSearchIfProcessorUnavailable(search) {
+  if (!search || (search.status !== 'queued' && search.status !== 'processing')) {
+    return search;
+  }
+
+  const processor = await getProcessorAvailability();
+  if (processor.available) {
+    return search;
+  }
+
+  return markSearchFailed(
+    redis,
+    search.id,
+    'PROCESSOR_UNAVAILABLE',
+    processor.message,
+    config.searchTtlSeconds
+  );
+}
+
+function logHealthFailure(details) {
+  const signature = JSON.stringify(details);
+  if (signature === lastHealthFailureSignature) {
+    return;
+  }
+
+  lastHealthFailureSignature = signature;
+  console.error(`[FaceAI] Health check failed ${signature}`);
+}
+
+function clearHealthFailure() {
+  if (!lastHealthFailureSignature) {
+    return;
+  }
+
+  lastHealthFailureSignature = null;
+  console.log('[FaceAI] Health check recovered');
+}
+
+async function getHealthStatus() {
+  const status = {
+    ok: true,
+    checks: {
+      redis: { ok: true },
+      processor: { ok: false, optional: true, ageMs: null, heartbeat: null }
+    }
+  };
+
+  try {
+    await redis.ping();
+  } catch (error) {
+    status.ok = false;
+    status.checks.redis = {
+      ok: false,
+      error: error.message
+    };
+  }
+
+  try {
+    const processor = await getProcessorAvailability();
+    status.checks.processor = {
+      ok: processor.available,
+      optional: true,
+      ageMs: processor.ageMs,
+      heartbeat: processor.heartbeat
+    };
+  } catch (error) {
+    status.checks.processor = {
+      ok: false,
+      optional: true,
+      error: error.message,
+      heartbeat: null
+    };
+  }
+
+  return status;
+}
+
 function getFaceAiSession(req) {
   const sessionId = req.cookies[config.sessionCookieName];
   return sessionId ? getSession(sessionId) : null;
@@ -268,8 +363,17 @@ function renderLegacyRacePage({ raceId, lang = 'it', result = null }) {
   </html>`;
 }
 
-app.get('/health', (req, res) => {
-  res.json({ ok: true });
+app.get('/health', async (req, res) => {
+  const status = await getHealthStatus();
+
+  if (!status.ok) {
+    logHealthFailure(status);
+    res.status(500).json(status);
+    return;
+  }
+
+  clearHealthFailure();
+  res.json(status);
 });
 
 app.get('/dev/legacy/race', (req, res) => {
@@ -406,6 +510,24 @@ app.post('/api/searches', requireSession, enforceSearchRateLimit, upload.single(
       return;
     }
 
+    const processor = await getProcessorAvailability();
+    if (!processor.available) {
+      logFaceAiAccess('Identification blocked: processor unavailable', req, {
+        user: summarizeUser(req.faceaiSession.user),
+        race: summarizeRace(race),
+        processorAgeMs: processor.ageMs,
+        processorHeartbeat: processor.heartbeat
+      });
+      if (req.file?.path) {
+        await fsp.unlink(req.file.path).catch(() => {});
+      }
+      res.status(503).json({
+        error: processor.message,
+        code: 'PROCESSOR_UNAVAILABLE'
+      });
+      return;
+    }
+
     const activeSearchId = await getActiveSearchId(redis, userId);
 
     if (activeSearchId) {
@@ -491,7 +613,8 @@ app.post('/api/searches', requireSession, enforceSearchRateLimit, upload.single(
 });
 
 app.get('/api/searches/:id', requireSession, async (req, res) => {
-  const search = await getSearchRecord(redis, req.params.id);
+  const rawSearch = await getSearchRecord(redis, req.params.id);
+  const search = await failSearchIfProcessorUnavailable(rawSearch);
   if (!search || search.userId !== req.faceaiSession.user.id) {
     res.status(404).json({ error: 'Search not found' });
     return;

faceai/apps/frontend/src/components/FaceAiUploadPanel.vue

@@ -62,7 +62,8 @@ const emit = defineEmits([
   'drag-leave',
   'drop',
   'clear-file',
-  'submit-search'
+  'submit-search',
+  'return-to-legacy'
 ]);
 </script>
 
@@ -162,7 +163,7 @@ const emit = defineEmits([
         <button v-if="selectedFile" class="btn btn-warning" type="button" :disabled="!canStartSearch" @click="emit('submit-search')">
           {{ t('uploadButton') }}
         </button>
-        <a class="btn btn-light" :href="session.returnUrl">{{ t('backButton') }}</a>
+        <button class="btn btn-light" type="button" @click="emit('return-to-legacy')">{{ t('backButton') }}</button>
       </div>
 
       <div v-if="!selectedFile && canPickFile" class="faceai-subtle-note">

faceai/apps/frontend/src/composables/useFaceAiHome.js

@@ -44,6 +44,7 @@ const copy = {
     invalidImage: 'Seleziona un file immagine valido.',
     pollError: 'Impossibile leggere lo stato della ricerca.',
     searchFailed: 'La ricerca non è andata a buon fine.',
+    processorUnavailable: 'Il motore FaceAI non è disponibile in questo momento. Riprova tra poco.',
     redirectError: 'Impossibile generare il link di ritorno.',
     chooseSelfie: 'Seleziona un selfie prima di avviare la ricerca.',
     raceDataUnavailable: 'I dati FaceAI non sono disponibili per questa gara.',
@@ -94,6 +95,7 @@ const copy = {
     invalidImage: 'Select a valid image file.',
     pollError: 'Unable to read the search status.',
     searchFailed: 'The search failed.',
+    processorUnavailable: 'The FaceAI processor is temporarily unavailable. Please try again shortly.',
     redirectError: 'Unable to build the return link.',
     chooseSelfie: 'Choose a selfie before starting the search.',
     raceDataUnavailable: 'FaceAI data is not available for this race.',
@@ -110,6 +112,7 @@ const knownServerMessages = {
   'FaceAI is not available for this race.': 'unavailableDefault',
   'Unable to read search status.': 'pollError',
   'The search failed.': 'searchFailed',
+  'FaceAI processor is temporarily unavailable. Please try again shortly.': 'processorUnavailable',
   'Unable to build return URL.': 'redirectError',
   'Unable to create the search.': 'searchCreateError',
   'Choose a selfie before starting the search.': 'chooseSelfie'
@@ -122,6 +125,18 @@ function isInvalidRaceAvailability(availability) {
   return availability?.reasonCode === 'RACE_DIRECTORY_NOT_FOUND' || availability?.reasonCode === 'MISSING_RACE_STORAGE';
 }
 
+function buildLegacyReturnUrl(url) {
+  if (!url) {
+    return legacyHomeUrl;
+  }
+
+  try {
+    return new URL(url, window.location.href).toString();
+  } catch {
+    return legacyHomeUrl;
+  }
+}
+
 export function useFaceAiHome() {
   const session = ref(null);
   const loading = ref(true);
@@ -408,9 +423,10 @@ export function useFaceAiHome() {
   async function pollSearch(searchId) {
     const response = await fetch(`/api/searches/${searchId}`, { credentials: 'include' });
     if (!response.ok) {
-      errorMessage.value = t('pollError');
+      const payload = await response.json().catch(() => ({}));
+      errorMessage.value = localizeServerMessage(payload.error, 'pollError');
       isSubmitting.value = false;
-      logFaceAiDebug('Search polling failed', { searchId, status: response.status });
+      logFaceAiDebug('Search polling failed', { searchId, status: response.status, payload });
       return;
     }
 
@@ -493,6 +509,12 @@ export function useFaceAiHome() {
     pollSearch(payload.id);
   }
 
+  function returnToLegacy() {
+    const returnUrl = buildLegacyReturnUrl(session.value?.returnUrl);
+    logFaceAiDebug('Returning to legacy race page', { returnUrl });
+    window.location.replace(returnUrl);
+  }
+
   onMounted(loadSession);
   onBeforeUnmount(() => {
     if (pollTimer) {
@@ -523,6 +545,7 @@ export function useFaceAiHome() {
     onFileChange,
     openFilePicker,
     redirectUrl,
+    returnToLegacy,
     selectedFile,
     selectedFileSizeLabel,
     session,

faceai/apps/frontend/src/views/HomeView.vue

@@ -26,6 +26,7 @@ const {
   onFileChange,
   openFilePicker,
   redirectUrl,
+  returnToLegacy,
   selectedFile,
   selectedFileSizeLabel,
   session,
@@ -72,6 +73,7 @@ const {
             @drop="onDrop"
             @clear-file="clearSelectedFile"
             @submit-search="submitSearch"
+            @return-to-legacy="returnToLegacy"
           />
         </div>
       </div>

faceai/apps/processor/src/config.js

@@ -9,6 +9,8 @@ export const config = {
   logRoot: process.env.FACEAI_LOG_ROOT || path.join(process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', 'logs'),
   pklRoot: process.env.FACEAI_PKL_ROOT || '/data/pkl',
   matcherBinary: process.env.FACEAI_MATCHER_BINARY || '/app/bin/face_matcher',
+  processorHeartbeatIntervalMs: Number(process.env.FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS || 5 * 1000),
+  processorHeartbeatTtlSeconds: Number(process.env.FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS || 20),
   searchTtlSeconds: Number(process.env.FACEAI_SEARCH_TTL_SECONDS || 24 * 60 * 60),
   resultTtlSeconds: Number(process.env.FACEAI_RESULT_TTL_SECONDS || 24 * 60 * 60)
 };

faceai/apps/processor/src/worker.js

@@ -10,6 +10,7 @@ import {
   markSearchFailed,
   markSearchProcessing,
   releaseActiveSearchLock,
+  updateProcessorHeartbeat,
   storeResultRecord
 } from '../../backend/src/redis-store.js';
 import { parseMatcherCsv, resolvePklPath, runFaceMatcher } from './worker-utils.js';
@@ -34,6 +35,18 @@ async function ensureMatcherBinaryAvailable() {
 
 console.log(`FaceAI processor configured matcher binary: ${config.matcherBinary}`);
 
+async function publishProcessorHeartbeat() {
+  try {
+    await updateProcessorHeartbeat(connection, config.processorHeartbeatTtlSeconds, {
+      pid: process.pid,
+      queueName: config.queueName,
+      matcherBinary: config.matcherBinary
+    });
+  } catch (error) {
+    console.error('Unable to publish FaceAI processor heartbeat:', error);
+  }
+}
+
 function formatLogLine(message, details) {
   const timestamp = new Date().toISOString();
   if (details === undefined) {
@@ -167,6 +180,13 @@ async function processJob(job) {
 }
 
 await ensureMatcherBinaryAvailable();
+await publishProcessorHeartbeat();
+
+const heartbeatTimer = setInterval(() => {
+  publishProcessorHeartbeat();
+}, config.processorHeartbeatIntervalMs);
+
+heartbeatTimer.unref();
 
 const worker = new Worker(config.queueName, processJob, {
   connection,

faceai/docker-compose.yml

@@ -28,6 +28,7 @@ services:
       FACEAI_RUNTIME_ROOT: /data/runtime
       FACEAI_UPLOAD_ROOT: /data/runtime/uploads
       FACEAI_LOG_ROOT: /data/logs
+      FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS: 20000
     volumes:
       - .:/app
       - ./logs:/data/logs
@@ -37,7 +38,7 @@ services:
     ports:
       - "3001:3001"
     healthcheck:
-      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/health | grep -q '\"ok\":true'"]
+      test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3001/health').then(async (response) => { const payload = await response.json().catch(() => ({})); if (!response.ok || payload.ok !== true) { console.error(JSON.stringify(payload)); process.exit(1); } }).catch((error) => { console.error(error.stack || error.message); process.exit(1); })"]
       interval: 10s
       timeout: 5s
       retries: 6
@@ -71,6 +72,8 @@ services:
       FACEAI_PKL_ROOT: /data/pkl
       FACEAI_WORKER_CONCURRENCY: 2
       FACEAI_MATCHER_BINARY: /app/bin/face_matcher
+      FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS: 5000
+      FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS: 20
     volumes:
       - .:/app
       - ./logs:/data/logs

faceai/docker/processor.Dockerfile

@@ -7,9 +7,17 @@ RUN apt-get update \
 
 WORKDIR /app
 
+COPY faceai/package.json ./package.json
+COPY faceai/apps/frontend/package.json apps/frontend/package.json
+COPY faceai/apps/backend/package.json apps/backend/package.json
+COPY faceai/apps/processor/package.json apps/processor/package.json
+
+RUN npm install
+
 COPY faceai /app
 COPY bin/Face_Recognition_Unix/face_matcher /app/bin/face_matcher
 
 RUN chmod +x /app/bin/face_matcher
 
+ENV NODE_ENV=production
 ENV FACEAI_MATCHER_BINARY=/app/bin/face_matcher