maddo/Regalamiunsorriso
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Add processor heartbeat management and improve health check functionality
All checks were successful
Publish FaceAI Container / publish (push) Successful in 3m7s
Details

Browse source 
- Introduced processor heartbeat configuration in environment variables and Docker setup.
- Implemented heartbeat publishing in the processor worker.
- Enhanced health check endpoint to include processor availability status.
- Updated frontend components to handle processor unavailability messages.
- Added legacy return functionality in the upload panel.
This commit is contained in:
MaddoScientisto 2026-04-19 11:50:11 +02:00
commit 87d9238795
14 changed files with 292 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

3
faceai/.env.example
View file

@ -14,6 +14,9 @@ FACEAI_UPLOAD_ROOT=/data/runtime/uploads
FACEAI_LOG_ROOT=/data/logs
FACEAI_PKL_ROOT=/data/pkl
FACEAI_MATCHER_BINARY=/app/bin/face_matcher
FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS=20000
FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS=5000
FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS=20
LIVE_SITE_BASE_URL=https://www.regalamiunsorriso.it
LIVE_SITE_LOGIN_URL=https://www.regalamiunsorriso.it/login_clienti-it.html
LIVE_SITE_RACE_URL=https://www.regalamiunsorriso.it/42%20HALF%20MARATHON%20FIRENZE_gara-1018545---96-1.html

7
faceai/README.md
View file

@ -291,6 +291,7 @@ services:
FACEAI_UPLOAD_ROOT: /data/runtime/uploads
FACEAI_LOG_ROOT: /data/logs
FACEAI_PKL_ROOT: /data/pkl
FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS: 20000
FACEAI_ENABLE_LOCAL_LEGACY_STATIC: 0
volumes:
- /mnt/storage/data/faceai/runtime:/data/runtime
@ -299,7 +300,7 @@ services:
ports:
- "3001:3001"
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/health | grep -q '\"ok\":true'"]
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3001/health').then(async (response) => { const payload = await response.json().catch(() => ({})); if (!response.ok || payload.ok !== true) { console.error(JSON.stringify(payload)); process.exit(1); } }).catch((error) => { console.error(error.stack || error.message); process.exit(1); })"]
interval: 10s
timeout: 5s
retries: 6
@ -326,7 +327,9 @@ services:
FACEAI_RUNTIME_ROOT: /data/runtime
FACEAI_LOG_ROOT: /data/logs
FACEAI_PKL_ROOT: /data/pkl
FACEAI_MATCHER_BINARY: /opt/face-recognition/face_matcher
FACEAI_MATCHER_BINARY: /app/bin/face_matcher
FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS: 5000
FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS: 20
FACEAI_WORKER_CONCURRENCY: 2
FACEAI_WORKER_TIMEOUT_MS: 300000
volumes:

1
faceai/apps/backend/src/config.js
View file

@ -28,6 +28,7 @@ export const config = {
uploadRoot: process.env.FACEAI_UPLOAD_ROOT || path.join(process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', 'uploads'),
searchTtlSeconds: Number(process.env.FACEAI_SEARCH_TTL_SECONDS || 24 * 60 * 60),
resultTtlSeconds: Number(process.env.FACEAI_RESULT_TTL_SECONDS || 24 * 60 * 60),
processorHeartbeatGraceMs: Number(process.env.FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS || 20 * 1000),
rateLimitWindowSeconds: Number(process.env.FACEAI_RATE_LIMIT_WINDOW_SECONDS || 10 * 60),
rateLimitMaxRequests: Number(process.env.FACEAI_RATE_LIMIT_MAX_REQUESTS || 5)
};

19
faceai/apps/backend/src/redis-store.js
View file

@ -24,6 +24,10 @@ function rateLimitKey(userId) {
return `faceai:rate-limit:${userId}`;
}
function processorHeartbeatKey() {
return 'faceai:processor-heartbeat';
}
export async function incrementRateLimit(redis, userId, windowSeconds) {
const key = rateLimitKey(userId);
const count = await redis.incr(key);
@ -50,6 +54,21 @@ export async function getActiveSearchId(redis, userId) {
return redis.get(activeSearchKey(userId));
}
export async function updateProcessorHeartbeat(redis, ttlSeconds, payload = {}) {
const heartbeat = {
updatedAt: Date.now(),
...payload
};
await redis.set(processorHeartbeatKey(), JSON.stringify(heartbeat), 'EX', ttlSeconds);
return heartbeat;
}
export async function getProcessorHeartbeat(redis) {
const raw = await redis.get(processorHeartbeatKey());
return raw ? JSON.parse(raw) : null;
}
export async function createSearchRecord(redis, payload, ttlSeconds) {
const searchId = randomId('search');
const record = {

129
faceai/apps/backend/src/server.js
View file

@ -15,9 +15,11 @@ import {
createRedisConnection,
createSearchRecord,
getActiveSearchId,
getProcessorHeartbeat,
getResultRecord,
getSearchRecord,
incrementRateLimit,
markSearchFailed,
saveSearchRecord
} from './redis-store.js';
import { getSearchQueue } from './queue.js';
@ -28,6 +30,7 @@ const frontendDist = path.resolve(__dirname, '../../frontend/dist');
const app = express();
const redis = createRedisConnection(config.redisUrl);
const searchQueue = getSearchQueue({ queueName: config.queueName, connection: redis });
let lastHealthFailureSignature = null;
await fsp.mkdir(config.uploadRoot, { recursive: true });
@ -89,6 +92,98 @@ function logFaceAiAccess(event, req, details = {}) {
})}`);
}
async function getProcessorAvailability() {
const heartbeat = await getProcessorHeartbeat(redis);
const ageMs = heartbeat ? Date.now() - Number(heartbeat.updatedAt || 0) : null;
const available = Boolean(heartbeat) && Number.isFinite(ageMs) && ageMs <= config.processorHeartbeatGraceMs;
return {
available,
ageMs,
heartbeat,
message: available
? null
: 'FaceAI processor is temporarily unavailable. Please try again shortly.'
};
}
async function failSearchIfProcessorUnavailable(search) {
if (!search || (search.status !== 'queued' && search.status !== 'processing')) {
return search;
}
const processor = await getProcessorAvailability();
if (processor.available) {
return search;
}
return markSearchFailed(
redis,
search.id,
'PROCESSOR_UNAVAILABLE',
processor.message,
config.searchTtlSeconds
);
}
function logHealthFailure(details) {
const signature = JSON.stringify(details);
if (signature === lastHealthFailureSignature) {
return;
}
lastHealthFailureSignature = signature;
console.error(`[FaceAI] Health check failed ${signature}`);
}
function clearHealthFailure() {
if (!lastHealthFailureSignature) {
return;
}
lastHealthFailureSignature = null;
console.log('[FaceAI] Health check recovered');
}
async function getHealthStatus() {
const status = {
ok: true,
checks: {
redis: { ok: true },
processor: { ok: false, optional: true, ageMs: null, heartbeat: null }
}
};
try {
await redis.ping();
} catch (error) {
status.ok = false;
status.checks.redis = {
ok: false,
error: error.message
};
}
try {
const processor = await getProcessorAvailability();
status.checks.processor = {
ok: processor.available,
optional: true,
ageMs: processor.ageMs,
heartbeat: processor.heartbeat
};
} catch (error) {
status.checks.processor = {
ok: false,
optional: true,
error: error.message,
heartbeat: null
};
}
return status;
}
function getFaceAiSession(req) {
const sessionId = req.cookies[config.sessionCookieName];
return sessionId ? getSession(sessionId) : null;
@ -268,8 +363,17 @@ function renderLegacyRacePage({ raceId, lang = 'it', result = null }) {
</html>`;
}
app.get('/health', (req, res) => {
res.json({ ok: true });
app.get('/health', async (req, res) => {
const status = await getHealthStatus();
if (!status.ok) {
logHealthFailure(status);
res.status(500).json(status);
return;
}
clearHealthFailure();
res.json(status);
});
app.get('/dev/legacy/race', (req, res) => {
@ -406,6 +510,24 @@ app.post('/api/searches', requireSession, enforceSearchRateLimit, upload.single(
return;
}
const processor = await getProcessorAvailability();
if (!processor.available) {
logFaceAiAccess('Identification blocked: processor unavailable', req, {
user: summarizeUser(req.faceaiSession.user),
race: summarizeRace(race),
processorAgeMs: processor.ageMs,
processorHeartbeat: processor.heartbeat
});
if (req.file?.path) {
await fsp.unlink(req.file.path).catch(() => {});
}
res.status(503).json({
error: processor.message,
code: 'PROCESSOR_UNAVAILABLE'
});
return;
}
const activeSearchId = await getActiveSearchId(redis, userId);
if (activeSearchId) {
@ -491,7 +613,8 @@ app.post('/api/searches', requireSession, enforceSearchRateLimit, upload.single(
});
app.get('/api/searches/:id', requireSession, async (req, res) => {
const search = await getSearchRecord(redis, req.params.id);
const rawSearch = await getSearchRecord(redis, req.params.id);
const search = await failSearchIfProcessorUnavailable(rawSearch);
if (!search || search.userId !== req.faceaiSession.user.id) {
res.status(404).json({ error: 'Search not found' });
return;

5
faceai/apps/frontend/src/components/FaceAiUploadPanel.vue
View file

@ -62,7 +62,8 @@ const emit = defineEmits([
'drag-leave',
'drop',
'clear-file',
'submit-search'
'submit-search',
'return-to-legacy'
]);
</script>
@ -162,7 +163,7 @@ const emit = defineEmits([
<button v-if="selectedFile" class="btn btn-warning" type="button" :disabled="!canStartSearch" @click="emit('submit-search')">
{{ t('uploadButton') }}
</button>
<a class="btn btn-light" :href="session.returnUrl">{{ t('backButton') }}</a>
<button class="btn btn-light" type="button" @click="emit('return-to-legacy')">{{ t('backButton') }}</button>
</div>
<div v-if="!selectedFile && canPickFile" class="faceai-subtle-note">

27
faceai/apps/frontend/src/composables/useFaceAiHome.js
View file

@ -44,6 +44,7 @@ const copy = {
invalidImage: 'Seleziona un file immagine valido.',
pollError: 'Impossibile leggere lo stato della ricerca.',
searchFailed: 'La ricerca non è andata a buon fine.',
processorUnavailable: 'Il motore FaceAI non è disponibile in questo momento. Riprova tra poco.',
redirectError: 'Impossibile generare il link di ritorno.',
chooseSelfie: 'Seleziona un selfie prima di avviare la ricerca.',
raceDataUnavailable: 'I dati FaceAI non sono disponibili per questa gara.',
@ -94,6 +95,7 @@ const copy = {
invalidImage: 'Select a valid image file.',
pollError: 'Unable to read the search status.',
searchFailed: 'The search failed.',
processorUnavailable: 'The FaceAI processor is temporarily unavailable. Please try again shortly.',
redirectError: 'Unable to build the return link.',
chooseSelfie: 'Choose a selfie before starting the search.',
raceDataUnavailable: 'FaceAI data is not available for this race.',
@ -110,6 +112,7 @@ const knownServerMessages = {
'FaceAI is not available for this race.': 'unavailableDefault',
'Unable to read search status.': 'pollError',
'The search failed.': 'searchFailed',
'FaceAI processor is temporarily unavailable. Please try again shortly.': 'processorUnavailable',
'Unable to build return URL.': 'redirectError',
'Unable to create the search.': 'searchCreateError',
'Choose a selfie before starting the search.': 'chooseSelfie'
@ -122,6 +125,18 @@ function isInvalidRaceAvailability(availability) {
return availability?.reasonCode === 'RACE_DIRECTORY_NOT_FOUND' || availability?.reasonCode === 'MISSING_RACE_STORAGE';
}
function buildLegacyReturnUrl(url) {
if (!url) {
return legacyHomeUrl;
}
try {
return new URL(url, window.location.href).toString();
} catch {
return legacyHomeUrl;
}
}
export function useFaceAiHome() {
const session = ref(null);
const loading = ref(true);
@ -408,9 +423,10 @@ export function useFaceAiHome() {
async function pollSearch(searchId) {
const response = await fetch(`/api/searches/${searchId}`, { credentials: 'include' });
if (!response.ok) {
errorMessage.value = t('pollError');
const payload = await response.json().catch(() => ({}));
errorMessage.value = localizeServerMessage(payload.error, 'pollError');
isSubmitting.value = false;
logFaceAiDebug('Search polling failed', { searchId, status: response.status });
logFaceAiDebug('Search polling failed', { searchId, status: response.status, payload });
return;
}
@ -493,6 +509,12 @@ export function useFaceAiHome() {
pollSearch(payload.id);
}
function returnToLegacy() {
const returnUrl = buildLegacyReturnUrl(session.value?.returnUrl);
logFaceAiDebug('Returning to legacy race page', { returnUrl });
window.location.replace(returnUrl);
}
onMounted(loadSession);
onBeforeUnmount(() => {
if (pollTimer) {
@ -523,6 +545,7 @@ export function useFaceAiHome() {
onFileChange,
openFilePicker,
redirectUrl,
returnToLegacy,
selectedFile,
selectedFileSizeLabel,
session,

2
faceai/apps/frontend/src/views/HomeView.vue
View file

@ -26,6 +26,7 @@ const {
onFileChange,
openFilePicker,
redirectUrl,
returnToLegacy,
selectedFile,
selectedFileSizeLabel,
session,
@ -72,6 +73,7 @@ const {
@drop="onDrop"
@clear-file="clearSelectedFile"
@submit-search="submitSearch"
@return-to-legacy="returnToLegacy"
/>
</div>
</div>

2
faceai/apps/processor/src/config.js
View file

@ -9,6 +9,8 @@ export const config = {
logRoot: process.env.FACEAI_LOG_ROOT || path.join(process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', 'logs'),
pklRoot: process.env.FACEAI_PKL_ROOT || '/data/pkl',
matcherBinary: process.env.FACEAI_MATCHER_BINARY || '/app/bin/face_matcher',
processorHeartbeatIntervalMs: Number(process.env.FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS || 5 * 1000),
processorHeartbeatTtlSeconds: Number(process.env.FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS || 20),
searchTtlSeconds: Number(process.env.FACEAI_SEARCH_TTL_SECONDS || 24 * 60 * 60),
resultTtlSeconds: Number(process.env.FACEAI_RESULT_TTL_SECONDS || 24 * 60 * 60)
};

20
faceai/apps/processor/src/worker.js
View file

@ -10,6 +10,7 @@ import {
markSearchFailed,
markSearchProcessing,
releaseActiveSearchLock,
updateProcessorHeartbeat,
storeResultRecord
} from '../../backend/src/redis-store.js';
import { parseMatcherCsv, resolvePklPath, runFaceMatcher } from './worker-utils.js';
@ -34,6 +35,18 @@ async function ensureMatcherBinaryAvailable() {
console.log(`FaceAI processor configured matcher binary: ${config.matcherBinary}`);
async function publishProcessorHeartbeat() {
try {
await updateProcessorHeartbeat(connection, config.processorHeartbeatTtlSeconds, {
pid: process.pid,
queueName: config.queueName,
matcherBinary: config.matcherBinary
});
} catch (error) {
console.error('Unable to publish FaceAI processor heartbeat:', error);
}
}
function formatLogLine(message, details) {
const timestamp = new Date().toISOString();
if (details === undefined) {
@ -167,6 +180,13 @@ async function processJob(job) {
}
await ensureMatcherBinaryAvailable();
await publishProcessorHeartbeat();
const heartbeatTimer = setInterval(() => {
publishProcessorHeartbeat();
}, config.processorHeartbeatIntervalMs);
heartbeatTimer.unref();
const worker = new Worker(config.queueName, processJob, {
connection,

5
faceai/docker-compose.yml
View file

@ -28,6 +28,7 @@ services:
FACEAI_RUNTIME_ROOT: /data/runtime
FACEAI_UPLOAD_ROOT: /data/runtime/uploads
FACEAI_LOG_ROOT: /data/logs
FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS: 20000
volumes:
- .:/app
- ./logs:/data/logs
@ -37,7 +38,7 @@ services:
ports:
- "3001:3001"
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/health | grep -q '\"ok\":true'"]
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3001/health').then(async (response) => { const payload = await response.json().catch(() => ({})); if (!response.ok || payload.ok !== true) { console.error(JSON.stringify(payload)); process.exit(1); } }).catch((error) => { console.error(error.stack || error.message); process.exit(1); })"]
interval: 10s
timeout: 5s
retries: 6
@ -71,6 +72,8 @@ services:
FACEAI_PKL_ROOT: /data/pkl
FACEAI_WORKER_CONCURRENCY: 2
FACEAI_MATCHER_BINARY: /app/bin/face_matcher
FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS: 5000
FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS: 20
volumes:
- .:/app
- ./logs:/data/logs

8
faceai/docker/processor.Dockerfile
View file

@ -7,9 +7,17 @@ RUN apt-get update \
WORKDIR /app
COPY faceai/package.json ./package.json
COPY faceai/apps/frontend/package.json apps/frontend/package.json
COPY faceai/apps/backend/package.json apps/backend/package.json
COPY faceai/apps/processor/package.json apps/processor/package.json
RUN npm install
COPY faceai /app
COPY bin/Face_Recognition_Unix/face_matcher /app/bin/face_matcher
RUN chmod +x /app/bin/face_matcher
ENV NODE_ENV=production
ENV FACEAI_MATCHER_BINARY=/app/bin/face_matcher

52
stacks/faceai.yml
View file

@ -3,14 +3,20 @@ services:
image: forgejo.maddoscientisto.net/maddo/faceai-client:latest
container_name: regalami-faceai
restart: unless-stopped
command: sh -c "mkdir -p /data/logs && npm run start >> /data/logs/backend.log 2>&1"
command:
- node
- docker/run-with-log-file.mjs
- /data/logs/backend.log
- npm
- run
- start
environment:
NODE_ENV: production
PORT: 3001
FACEAI_FRONTEND_URL: https://ai.regalamiunsorriso.it
FACEAI_PUBLIC_BASE_URL: https://ai.regalamiunsorriso.it
FACEAI_LEGACY_RETURN_URL: https://www.regalamiunsorriso.it/faceai_return.php
FACEAI_LEGACY_HOME_URL: https://www.regalamiunsorriso.it/
FACEAI_LEGACY_HOME_URL: https://www.regalamiunsorriso.it
FACEAI_SHARED_SECRET: disagio-spaghetti-science-lol-boh
FACEAI_SESSION_COOKIE: rus_faceai_session
FACEAI_REDIS_URL: redis://redis:6379
@ -19,21 +25,35 @@ services:
FACEAI_UPLOAD_ROOT: /data/runtime/uploads
FACEAI_LOG_ROOT: /data/logs
FACEAI_PKL_ROOT: /data/pkl
FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS: 20000
FACEAI_ENABLE_LOCAL_LEGACY_STATIC: 0
volumes:
- /var/docker/faceai/runtime:/data/runtime
- /var/docker/faceai/logs:/data/logs
- /mnt/storage/data/faceai/runtime:/data/runtime
- /mnt/storage/data/faceai/logs:/data/logs
- /mnt/nas12/nas2/RUS:/data/pkl:ro
ports:
- "127.0.0.1:3001:3001"
- "3001:3001"
healthcheck:
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3001/health').then(async (response) => { const payload = await response.json().catch(() => ({})); if (!response.ok || payload.ok !== true) { console.error(JSON.stringify(payload)); process.exit(1); } }).catch((error) => { console.error(error.stack || error.message); process.exit(1); })"]
interval: 10s
timeout: 5s
retries: 6
start_period: 20s
depends_on:
- redis
redis:
condition: service_healthy
processor:
image: forgejo.maddoscientisto.net/maddo/faceai-processor:latest
container_name: regalami-faceai-processor
restart: unless-stopped
command: sh -c "mkdir -p /data/logs && npm run start:processor >> /data/logs/processor.log 2>&1"
command:
- node
- docker/run-with-log-file.mjs
- /data/logs/processor.log
- npm
- run
- start:processor
environment:
NODE_ENV: production
FACEAI_REDIS_URL: redis://redis:6379
@ -41,18 +61,26 @@ services:
FACEAI_RUNTIME_ROOT: /data/runtime
FACEAI_LOG_ROOT: /data/logs
FACEAI_PKL_ROOT: /data/pkl
FACEAI_MATCHER_BINARY: /opt/face-recognition/face_matcher
FACEAI_WORKER_CONCURRENCY: 2
FACEAI_MATCHER_BINARY: /app/bin/face_matcher
FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS: 5000
FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS: 20
FACEAI_WORKER_CONCURRENCY: 8
FACEAI_WORKER_TIMEOUT_MS: 300000
volumes:
- /var/docker/faceai/runtime:/data/runtime
- /var/docker/faceai/logs:/data/logs
- /mnt/storage/data/faceai/runtime:/data/runtime
- /mnt/storage/data/faceai/logs:/data/logs
- /mnt/nas12/nas2/RUS:/data/pkl:ro
depends_on:
- redis
redis:
condition: service_healthy
redis:
image: redis:7-alpine
container_name: regalami-faceai-redis
restart: unless-stopped
command: redis-server --appendonly no
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 12

33
www/_js/rus-ecom-240621.js
View file

@ -304,6 +304,34 @@ function launchFaceAi() {
return false;
}
function clearLegacyLoadingState() {
$("body").removeClass("loading");
}
function shouldClearLegacyLoadingOnRestore(event) {
if (event && event.persisted) {
return true;
}
if (window.performance && typeof window.performance.getEntriesByType === "function") {
var entries = window.performance.getEntriesByType("navigation");
if (entries && entries.length && entries[0].type === "back_forward") {
return true;
}
}
return false;
}
function restoreLegacyPageAfterHistoryNavigation(event) {
if (!shouldClearLegacyLoadingOnRestore(event)) {
return;
}
clearLegacyLoadingState();
logFaceAiDebug("Cleared legacy loading state after history navigation");
}
function initFaceAiRaceSearchButton() {
var select = $("#tipoPuntoFoto");
if (!select.length || $("#faceaiLaunchButton").length || !faceAiFeatureEnabled()) {
@ -529,11 +557,16 @@ function goPage()
}
$(function() {
clearLegacyLoadingState();
initFaceAiRaceSearchButton();
initFaceAiErrorModal();
logFaceAiDebug("Legacy race page ready");
});
if (window.addEventListener) {
window.addEventListener("pageshow", restoreLegacyPageAfterHistoryNavigation);
}