From a026fec62bae4e94b5e6b9c9708d2e5ae0def6ab Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sat, 9 May 2026 14:46:44 +0200 Subject: [PATCH] Added support for new faceai parameters --- .../run_face_encoder.ps1 | 117 ++++++++++++++++-- faceai/.env.development | 1 + faceai/.env.example | 1 + faceai/.env.production | 1 + faceai/README.md | 2 + faceai/apps/processor/src/config.js | 20 +++ faceai/apps/processor/src/worker-utils.js | 12 +- faceai/apps/processor/src/worker.js | 5 + faceai/docker-compose.override.yml | 1 + faceai/docker-compose.yml | 1 + faceai/docker/processor.Dockerfile | 3 +- stacks/faceai.yml | 1 + 12 files changed, 152 insertions(+), 13 deletions(-) diff --git a/bin/Face_Recognition_Windows/run_face_encoder.ps1 b/bin/Face_Recognition_Windows/run_face_encoder.ps1 index 291162f9..9d935a5d 100644 --- a/bin/Face_Recognition_Windows/run_face_encoder.ps1 +++ b/bin/Face_Recognition_Windows/run_face_encoder.ps1 @@ -190,6 +190,99 @@ function Get-MulticoreSetting { return -1 } +function Get-ParallelismSetting { + param( + [Parameter(Mandatory = $true)] + [ValidateSet('cpu', 'gpu')] + [string]$EncoderVariant + ) + + Write-Host '' + + if ($EncoderVariant -eq 'gpu') { + Write-Host "Seleziona il livello di multiprocess per l'elaborazione GPU:" + Write-Host ' 1 = 1/8 dei worker' + Write-Host ' 2 = 1/4 dei worker' + Write-Host ' 3 = 1/2 dei worker (predefinito)' + Write-Host ' 4 = 3/4 dei worker' + Write-Host ' 5 = tutti i worker' + Write-Host '' + + $parallelismChoice = Read-Host 'Inserisci il livello (1-5) oppure premi Invio per usare il predefinito (3)' + } else { + Write-Host "Seleziona il livello di multicore per l'elaborazione CPU:" + Write-Host ' 1 = 1/8 dei core' + Write-Host ' 2 = 1/4 dei core' + Write-Host ' 3 = 1/2 dei core (predefinito)' + Write-Host ' 4 = 3/4 dei core' + Write-Host ' 5 = n-2 core' + Write-Host '' + + $parallelismChoice = Read-Host 'Inserisci il livello (1-5) oppure premi Invio per usare il predefinito (3)' + } + + if ($parallelismChoice -match '^[1-5]$') { + return [int]$parallelismChoice + } + + return 3 +} + +function Get-MinSizeSetting { + Write-Host '' + Write-Host 'Imposta la dimensione minima in pixel del volto da considerare.' + Write-Host 'Premi Invio per usare il predefinito (35).' + Write-Host '' + + $minSizeChoice = Read-Host 'Min size' + if ([string]::IsNullOrWhiteSpace($minSizeChoice)) { + return 35 + } + + if ($minSizeChoice -match '^\d+$' -and [int]$minSizeChoice -gt 0) { + return [int]$minSizeChoice + } + + Write-Warning 'Valore non valido. Viene usato il valore predefinito 35.' + return 35 +} + +function Get-UpsampleSetting { + param( + [Parameter(Mandatory = $true)] + [ValidateSet('cpu', 'gpu')] + [string]$EncoderVariant + ) + + $defaultEnabled = $EncoderVariant -eq 'cpu' + $defaultLabel = if ($defaultEnabled) { 'S' } else { 'N' } + + Write-Host '' + Write-Host 'Vuoi abilitare l''upsample per migliorare il rilevamento dei volti piccoli?' + if ($defaultEnabled) { + Write-Host 'Premi Invio per usare il predefinito consigliato per CPU (S).' + } else { + Write-Host 'Premi Invio per usare il predefinito consigliato per GPU (N).' + } + Write-Host '' + + $upsampleChoice = Read-Host "Abilitare upsample? (S/N, predefinito: $defaultLabel)" + if ([string]::IsNullOrWhiteSpace($upsampleChoice)) { + return $defaultEnabled + } + + if ($upsampleChoice -match '^(s|si|y|yes)$') { + return $true + } + + if ($upsampleChoice -match '^(n|no)$') { + return $false + } + + Write-Warning "Scelta non valida. Viene usato il predefinito: $defaultLabel" + return $defaultEnabled +} + function Show-PklFilePicker { param( [Parameter(Mandatory = $true)] @@ -343,11 +436,9 @@ function Invoke-FaceEncoding { $encoderVariant = Get-EncoderVariant $encoderDir = Join-Path $PSScriptRoot ("face_encoder_{0}" -f $encoderVariant) $outputDir = Join-Path $encoderDir 'output' - $multicore = -1 - - if ($encoderVariant -eq 'cpu') { - $multicore = Get-MulticoreSetting - } + $parallelism = Get-ParallelismSetting -EncoderVariant $encoderVariant + $minSize = Get-MinSizeSetting + $useUpsample = Get-UpsampleSetting -EncoderVariant $encoderVariant $inputPath = [ModernFolderPicker]::Show('Select the folder containing images to encode') @@ -380,17 +471,25 @@ function Invoke-FaceEncoding { $encoderArgs.Add('-l') $encoderArgs.Add($logFile) - if ($multicore -ge 0) { - $encoderArgs.Add('-m') - $encoderArgs.Add([string]$multicore) + $encoderArgs.Add('-m') + $encoderArgs.Add([string]$parallelism) + $encoderArgs.Add('-s') + $encoderArgs.Add([string]$minSize) + + if ($useUpsample) { + $encoderArgs.Add('-u') } Write-Host "Input folder : $($inputFolder.FullName)" Write-Host "Encoder : $($encoderVariant.ToUpperInvariant())" Write-Host "Race name : $raceName" if ($encoderVariant -eq 'cpu') { - Write-Host "Multicore : $(if ($multicore -ge 0) { $multicore } else { 'default (3)' })" + Write-Host "Multicore : $parallelism" + } else { + Write-Host "Multiprocess : $parallelism" } + Write-Host "Min size : $minSize" + Write-Host "Upsample : $(if ($useUpsample) { 'enabled' } else { 'disabled' })" Write-Host "Output file : $outputFile" Write-Host "Log file : $logFile" Write-Host "Command : $encoderExe $encoderArgs" diff --git a/faceai/.env.development b/faceai/.env.development index 2ddbadd2..cf9786a9 100644 --- a/faceai/.env.development +++ b/faceai/.env.development @@ -18,6 +18,7 @@ FACEAI_UPLOAD_ROOT=/data/runtime/uploads FACEAI_LOG_ROOT=/data/logs FACEAI_PKL_ROOT=/data/pkl FACEAI_MATCHER_BINARY=/app/bin/face_matcher +FACEAI_MATCHER_TOLERANCE=0.5 FACEAI_WORKER_CONCURRENCY=2 FACEAI_WORKER_TIMEOUT_MS=300000 FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS=20000 diff --git a/faceai/.env.example b/faceai/.env.example index 20fd0a9c..3db21884 100644 --- a/faceai/.env.example +++ b/faceai/.env.example @@ -33,6 +33,7 @@ FACEAI_RUNTIME_BIND=/mnt/storage/data/faceai/runtime FACEAI_LOG_BIND=/mnt/storage/data/faceai/logs FACEAI_PKL_BIND=/mnt/nas12/nas2/RUS FACEAI_MATCHER_BINARY=/app/bin/face_matcher +FACEAI_MATCHER_TOLERANCE=0.5 FACEAI_WORKER_CONCURRENCY=8 FACEAI_WORKER_TIMEOUT_MS=300000 FACEAI_PROCESSOR_HEARTBEAT_GRACE_MS=20000 diff --git a/faceai/.env.production b/faceai/.env.production index 983d424d..b0ff931b 100644 --- a/faceai/.env.production +++ b/faceai/.env.production @@ -23,6 +23,7 @@ FACEAI_RUNTIME_BIND=/mnt/storage/data/faceai/runtime FACEAI_LOG_BIND=/mnt/storage/data/faceai/logs FACEAI_PKL_BIND=/mnt/nas12/nas2/RUS FACEAI_MATCHER_BINARY=/app/bin/face_matcher +FACEAI_MATCHER_TOLERANCE=0.5 FACEAI_WORKER_CONCURRENCY=8 FACEAI_WORKER_TIMEOUT_MS=300000 FACEAI_ENABLE_LOCAL_LEGACY_STATIC=0 \ No newline at end of file diff --git a/faceai/README.md b/faceai/README.md index d48daa2b..4dd5cb9c 100644 --- a/faceai/README.md +++ b/faceai/README.md @@ -328,6 +328,7 @@ Processor settings: | --- | --- | --- | --- | | `FACEAI_PKL_ROOT` | yes | `/data/pkl` | mounted race-to-PKL dataset root | | `FACEAI_MATCHER_BINARY` | yes | `/app/bin/face_matcher` | matcher executable baked into the processor image | +| `FACEAI_MATCHER_TOLERANCE` | optional | `0.5` | forwarded to `face_matcher --tollerance`; must stay between `0.35` and `0.75` | | `FACEAI_WORKER_CONCURRENCY` | optional | `2` | BullMQ worker concurrency | | `FACEAI_WORKER_TIMEOUT_MS` | optional | `300000` | matcher timeout in milliseconds | @@ -432,6 +433,7 @@ FACEAI_UPLOAD_ROOT=/data/runtime/uploads FACEAI_LOG_ROOT=/data/logs FACEAI_PKL_ROOT=/data/pkl FACEAI_MATCHER_BINARY=/app/bin/face_matcher +FACEAI_MATCHER_TOLERANCE=0.5 ``` If you want FaceAI to force the older bridge-style return path instead, set `FACEAI_LEGACY_RETURN_MODE=bridge` and point `FACEAI_LEGACY_RETURN_URL` at the appropriate legacy bridge endpoint. diff --git a/faceai/apps/processor/src/config.js b/faceai/apps/processor/src/config.js index 11763c8f..80de646f 100644 --- a/faceai/apps/processor/src/config.js +++ b/faceai/apps/processor/src/config.js @@ -1,5 +1,24 @@ import path from 'node:path'; +function readOptionalNumberEnv(name) { + const raw = process.env[name]; + if (raw === undefined || raw === '') { + return null; + } + + const value = Number(raw); + if (Number.isNaN(value)) { + throw new Error(`${name} must be a number when set`); + } + + return value; +} + +const matcherTolerance = readOptionalNumberEnv('FACEAI_MATCHER_TOLERANCE'); +if (matcherTolerance !== null && (matcherTolerance < 0.35 || matcherTolerance > 0.75)) { + throw new Error('FACEAI_MATCHER_TOLERANCE must be between 0.35 and 0.75'); +} + export const config = { redisUrl: process.env.FACEAI_REDIS_URL || 'redis://redis:6379', queueName: process.env.FACEAI_QUEUE_NAME || 'faceai-searches', @@ -9,6 +28,7 @@ export const config = { logRoot: process.env.FACEAI_LOG_ROOT || path.join(process.env.FACEAI_RUNTIME_ROOT || '/data/runtime', 'logs'), pklRoot: process.env.FACEAI_PKL_ROOT || '/data/pkl', matcherBinary: process.env.FACEAI_MATCHER_BINARY || '/app/bin/face_matcher', + matcherTolerance, processorHeartbeatIntervalMs: Number(process.env.FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS || 5 * 1000), processorHeartbeatTtlSeconds: Number(process.env.FACEAI_PROCESSOR_HEARTBEAT_TTL_SECONDS || 60), searchTtlSeconds: Number(process.env.FACEAI_SEARCH_TTL_SECONDS || 24 * 60 * 60), diff --git a/faceai/apps/processor/src/worker-utils.js b/faceai/apps/processor/src/worker-utils.js index e86c92b2..5cb01fbe 100644 --- a/faceai/apps/processor/src/worker-utils.js +++ b/faceai/apps/processor/src/worker-utils.js @@ -19,17 +19,23 @@ export async function resolvePklPath({ raceId, raceStorage, pklRoot }) { return availability.pklPath; } -export async function runFaceMatcher({ matcherBinary, selfiePath, pklPath, csvPath, logPath, timeoutMs }) { +export async function runFaceMatcher({ matcherBinary, matcherTolerance, selfiePath, pklPath, csvPath, logPath, timeoutMs }) { await fs.mkdir(path.dirname(csvPath), { recursive: true }); await fs.mkdir(path.dirname(logPath), { recursive: true }); return new Promise((resolve, reject) => { - const child = spawn(matcherBinary, [ + const matcherArgs = [ '--image', selfiePath, '--encodings', pklPath, '--out', csvPath, '--log', logPath - ], { + ]; + + if (matcherTolerance !== null && matcherTolerance !== undefined) { + matcherArgs.push('--tollerance', String(matcherTolerance)); + } + + const child = spawn(matcherBinary, matcherArgs, { stdio: 'ignore' }); diff --git a/faceai/apps/processor/src/worker.js b/faceai/apps/processor/src/worker.js index fcafaeee..c42ab964 100644 --- a/faceai/apps/processor/src/worker.js +++ b/faceai/apps/processor/src/worker.js @@ -34,6 +34,9 @@ async function ensureMatcherBinaryAvailable() { } console.log(`FaceAI processor configured matcher binary: ${config.matcherBinary}`); +if (config.matcherTolerance !== null) { + console.log(`FaceAI processor configured matcher tolerance: ${config.matcherTolerance}`); +} async function publishProcessorHeartbeat() { try { @@ -138,6 +141,7 @@ async function processJob(job) { await appendSearchLog(searchLogPath, 'Running matcher', { matcherBinary: config.matcherBinary, + matcherTolerance: config.matcherTolerance, csvPath, matcherLogPath: logPath, timeoutMs: config.workerTimeoutMs @@ -146,6 +150,7 @@ async function processJob(job) { try { await runFaceMatcher({ matcherBinary: config.matcherBinary, + matcherTolerance: config.matcherTolerance, selfiePath: search.selfiePath, pklPath, csvPath, diff --git a/faceai/docker-compose.override.yml b/faceai/docker-compose.override.yml index 038f103f..562f3f84 100644 --- a/faceai/docker-compose.override.yml +++ b/faceai/docker-compose.override.yml @@ -60,6 +60,7 @@ services: FACEAI_LOG_ROOT: ${FACEAI_LOG_ROOT:-/data/logs} FACEAI_PKL_ROOT: ${FACEAI_PKL_ROOT:-/data/pkl} FACEAI_MATCHER_BINARY: ${FACEAI_MATCHER_BINARY:-/app/bin/face_matcher} + FACEAI_MATCHER_TOLERANCE: ${FACEAI_MATCHER_TOLERANCE:-0.5} FACEAI_WORKER_CONCURRENCY: ${FACEAI_WORKER_CONCURRENCY:-2} FACEAI_WORKER_TIMEOUT_MS: ${FACEAI_WORKER_TIMEOUT_MS:-300000} FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS: ${FACEAI_PROCESSOR_HEARTBEAT_INTERVAL_MS:-5000} diff --git a/faceai/docker-compose.yml b/faceai/docker-compose.yml index 0aba1a99..b9c4ff7b 100644 --- a/faceai/docker-compose.yml +++ b/faceai/docker-compose.yml @@ -61,6 +61,7 @@ services: FACEAI_LOG_ROOT: ${FACEAI_LOG_ROOT:-/data/logs} FACEAI_PKL_ROOT: ${FACEAI_PKL_ROOT:-/data/pkl} FACEAI_MATCHER_BINARY: ${FACEAI_MATCHER_BINARY:-/app/bin/face_matcher} + FACEAI_MATCHER_TOLERANCE: ${FACEAI_MATCHER_TOLERANCE:-0.5} FACEAI_WORKER_CONCURRENCY: ${FACEAI_WORKER_CONCURRENCY:-8} FACEAI_WORKER_TIMEOUT_MS: ${FACEAI_WORKER_TIMEOUT_MS:-300000} volumes: diff --git a/faceai/docker/processor.Dockerfile b/faceai/docker/processor.Dockerfile index 958c1e0e..f43a31e1 100644 --- a/faceai/docker/processor.Dockerfile +++ b/faceai/docker/processor.Dockerfile @@ -20,4 +20,5 @@ COPY bin/Face_Recognition_Unix/face_matcher /app/bin/face_matcher RUN chmod +x /app/bin/face_matcher ENV NODE_ENV=production -ENV FACEAI_MATCHER_BINARY=/app/bin/face_matcher \ No newline at end of file +ENV FACEAI_MATCHER_BINARY=/app/bin/face_matcher +ENV FACEAI_MATCHER_TOLERANCE=0.5 \ No newline at end of file diff --git a/stacks/faceai.yml b/stacks/faceai.yml index dcf526b2..aa38b676 100644 --- a/stacks/faceai.yml +++ b/stacks/faceai.yml @@ -61,6 +61,7 @@ services: FACEAI_LOG_ROOT: /data/logs FACEAI_PKL_ROOT: /data/pkl FACEAI_MATCHER_BINARY: /app/bin/face_matcher + FACEAI_MATCHER_TOLERANCE: 0.5 FACEAI_WORKER_CONCURRENCY: 8 FACEAI_WORKER_TIMEOUT_MS: 300000 volumes: