MaddoScientisto
32db61c381
All checks were successful
Publish FaceAI Container / publish (push) Successful in 13m22s
- Added configuration options for audit database path and retention days in backend and processor. - Integrated audit logging in server and worker processes to track search requests, completions, and failures. - Created utility functions for reading and parsing audit logs in end-to-end tests. - Updated Docker Compose files to include audit database configuration. - Added new tests to verify audit log entries for successful and no-results searches.
478 lines
No EOL
14 KiB
JavaScript
478 lines
No EOL
14 KiB
JavaScript
const fs = require('node:fs/promises');
|
|
const fsSync = require('node:fs');
|
|
const path = require('node:path');
|
|
const { spawn, spawnSync } = require('node:child_process');
|
|
const { DatabaseSync } = require('node:sqlite');
|
|
|
|
const ROOT_DIR = path.resolve(__dirname, '..', '..');
|
|
const WORKSPACE_ROOT = path.resolve(ROOT_DIR, '..');
|
|
const LOG_ROOT = path.join(ROOT_DIR, 'logs');
|
|
const SEARCH_LOG_ROOT = path.join(LOG_ROOT, 'searches');
|
|
const AUDIT_DB_PATH = path.join(LOG_ROOT, 'faceai-audit.sqlite');
|
|
const PREFER_CONTAINER_AUDIT_DB = process.env.FACEAI_E2E_AUDIT_READ_FROM_CONTAINER === '1' || process.platform === 'win32';
|
|
const AUDIT_DB_PATH_IN_CONTAINER = process.env.FACEAI_E2E_AUDIT_DB_PATH_IN_CONTAINER || '/data/runtime/faceai-audit.sqlite';
|
|
const AUDIT_DB_QUERY_CONTAINER = process.env.FACEAI_E2E_AUDIT_QUERY_CONTAINER || 'regalami-faceai-processor';
|
|
const FACEAI_BASE_URL = process.env.FACEAI_E2E_BASE_URL || 'http://127.0.0.1:3001';
|
|
const SIMULATOR_URL = process.env.FACEAI_E2E_SIMULATOR_URL || 'http://127.0.0.1:8080/Foto2.abl?id_gara=1018547&pageRow=96&pageNumber=1';
|
|
const LEGACY_BASE_URL = process.env.FACEAI_E2E_LEGACY_BASE_URL || 'http://127.0.0.1:8080';
|
|
const LEGACY_HOME_URL = process.env.FACEAI_E2E_LEGACY_HOME_URL || `${LEGACY_BASE_URL}/index.jsp`;
|
|
const LEGACY_LOGIN_URL = process.env.FACEAI_E2E_LEGACY_LOGIN_URL || `${LEGACY_BASE_URL}/login_clienti.html`;
|
|
const LEGACY_USERNAME = process.env.FACEAI_E2E_LEGACY_USERNAME || 'test';
|
|
const LEGACY_PASSWORD = process.env.FACEAI_E2E_LEGACY_PASSWORD || 'test1';
|
|
const SELFIE_NAME = process.env.FACEAI_E2E_SELFIE || '2026/04.APRILE/ISOLOTTO/01.FOTO/CKL_8459.JPG';
|
|
const EXPECTED_MATCH_COUNT = Number(process.env.FACEAI_E2E_EXPECTED_MATCH_COUNT || '6');
|
|
const LEGACY_RACE_ID = process.env.FACEAI_E2E_RACE_ID || '1018547';
|
|
|
|
function quoteShellArg(value) {
|
|
if (!/[\s"]/u.test(value)) {
|
|
return value;
|
|
}
|
|
|
|
return `"${value.replace(/"/g, '\\"')}"`;
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => {
|
|
setTimeout(resolve, ms);
|
|
});
|
|
}
|
|
|
|
function runCommand(command, args, options = {}) {
|
|
const { cwd = ROOT_DIR, allowFailure = false } = options;
|
|
const executable = process.platform === 'win32' && command === 'npm' ? 'npm.cmd' : command;
|
|
const useShell = process.platform === 'win32';
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const child = useShell
|
|
? spawn([executable, ...args].map(quoteShellArg).join(' '), {
|
|
cwd,
|
|
env: process.env,
|
|
shell: true,
|
|
stdio: ['ignore', 'pipe', 'pipe']
|
|
})
|
|
: spawn(executable, args, {
|
|
cwd,
|
|
env: process.env,
|
|
stdio: ['ignore', 'pipe', 'pipe']
|
|
});
|
|
|
|
let stdout = '';
|
|
let stderr = '';
|
|
|
|
child.stdout.on('data', (chunk) => {
|
|
stdout += chunk.toString();
|
|
});
|
|
|
|
child.stderr.on('data', (chunk) => {
|
|
stderr += chunk.toString();
|
|
});
|
|
|
|
child.on('error', reject);
|
|
child.on('close', (code) => {
|
|
const result = { code, stdout, stderr };
|
|
if (code === 0 || allowFailure) {
|
|
resolve(result);
|
|
return;
|
|
}
|
|
|
|
const error = new Error(`Command failed: ${executable} ${args.join(' ')}`);
|
|
error.result = result;
|
|
reject(error);
|
|
});
|
|
});
|
|
}
|
|
|
|
function dockerCompose(args, options) {
|
|
return runCommand('docker', ['compose', ...args], options);
|
|
}
|
|
|
|
async function prepareHostState() {
|
|
await fs.rm(LOG_ROOT, { recursive: true, force: true });
|
|
await fs.mkdir(LOG_ROOT, { recursive: true });
|
|
}
|
|
|
|
async function waitForHttp(url, validate, timeoutMs = 3 * 60 * 1000) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let lastError = null;
|
|
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
const response = await fetch(url);
|
|
const bodyText = await response.text();
|
|
let parsedBody = null;
|
|
|
|
try {
|
|
parsedBody = JSON.parse(bodyText);
|
|
} catch {
|
|
parsedBody = null;
|
|
}
|
|
|
|
if (validate({ response, bodyText, parsedBody })) {
|
|
return;
|
|
}
|
|
|
|
lastError = new Error(`Readiness check did not pass for ${url}.`);
|
|
} catch (error) {
|
|
lastError = error;
|
|
}
|
|
|
|
await sleep(1000);
|
|
}
|
|
|
|
throw lastError || new Error(`Timed out waiting for ${url}`);
|
|
}
|
|
|
|
function getSelfiePath(fileName = SELFIE_NAME) {
|
|
const relativeSegments = fileName.split(/[\\/]+/u);
|
|
if (relativeSegments.length > 1) {
|
|
return path.join(WORKSPACE_ROOT, 'test_pkl', ...relativeSegments);
|
|
}
|
|
|
|
return path.join(WORKSPACE_ROOT, 'test_pkl', 'test_images', fileName);
|
|
}
|
|
|
|
function buildSimulatorUrl({
|
|
raceId = LEGACY_RACE_ID,
|
|
lang = 'it',
|
|
raceSlug = 'isolotto',
|
|
raceName = 'Festa sociale UP Isolotto',
|
|
raceYear = '2026',
|
|
raceMonthFolder = '04.APRILE',
|
|
raceFolder = 'ISOLOTTO',
|
|
pageRow = '96',
|
|
pageNumber = '1'
|
|
} = {}) {
|
|
const url = new URL('/Foto2.abl', LEGACY_BASE_URL);
|
|
url.searchParams.set('id_gara', raceId);
|
|
url.searchParams.set('pageRow', pageRow);
|
|
url.searchParams.set('pageNumber', pageNumber);
|
|
url.searchParams.set('lang', lang);
|
|
return url.toString();
|
|
}
|
|
|
|
function buildHandoffUrl({
|
|
raceId = LEGACY_RACE_ID,
|
|
lang = 'it',
|
|
raceSlug = 'isolotto',
|
|
raceName = 'Festa sociale UP Isolotto',
|
|
raceYear = '2026',
|
|
raceMonthFolder = '04.APRILE',
|
|
raceFolder = 'ISOLOTTO',
|
|
userId = '1',
|
|
displayName = `Local Test User ${userId}`,
|
|
email = `local-test-${userId}@example.invalid`,
|
|
membershipStatus = 'active',
|
|
returnUrl = buildSimulatorUrl({ raceId, lang, raceSlug, raceName, raceYear, raceMonthFolder, raceFolder })
|
|
} = {}) {
|
|
const url = new URL('/dev/legacy/launch', FACEAI_BASE_URL);
|
|
url.searchParams.set('raceId', raceId);
|
|
url.searchParams.set('raceSlug', raceSlug);
|
|
url.searchParams.set('raceName', raceName);
|
|
url.searchParams.set('raceYear', raceYear);
|
|
url.searchParams.set('raceMonthFolder', raceMonthFolder);
|
|
url.searchParams.set('raceFolder', raceFolder);
|
|
url.searchParams.set('lang', lang);
|
|
url.searchParams.set('returnUrl', returnUrl);
|
|
url.searchParams.set('devUserId', userId);
|
|
url.searchParams.set('devDisplayName', displayName);
|
|
url.searchParams.set('devEmail', email);
|
|
url.searchParams.set('devMembershipStatus', membershipStatus);
|
|
return url.toString();
|
|
}
|
|
|
|
function buildLegacyLoginFormData() {
|
|
if (!LEGACY_USERNAME || !LEGACY_PASSWORD) {
|
|
throw new Error('FACEAI_E2E_LEGACY_USERNAME and FACEAI_E2E_LEGACY_PASSWORD must be set before running authenticated local E2E checks.');
|
|
}
|
|
|
|
return {
|
|
login: LEGACY_USERNAME,
|
|
pwd: LEGACY_PASSWORD,
|
|
cmdIU: 'check',
|
|
act: '',
|
|
thePage: ''
|
|
};
|
|
}
|
|
|
|
async function performLocalLoginRequest(requestContext) {
|
|
const response = await requestContext.post(`${LEGACY_BASE_URL}/Logon.abl`, {
|
|
form: buildLegacyLoginFormData(),
|
|
failOnStatusCode: false
|
|
});
|
|
|
|
const finalUrl = response.url();
|
|
const bodyText = await response.text();
|
|
|
|
if (!response.ok()) {
|
|
throw new Error(`Local login request failed with HTTP ${response.status()} at ${finalUrl}`);
|
|
}
|
|
|
|
if (/login_clienti|Username \/ Email|Password/iu.test(bodyText) && !/user_logout|dettaglio_clienti|Il mio account/iu.test(bodyText)) {
|
|
throw new Error(`Local login request appears to have remained on the login page at ${finalUrl}`);
|
|
}
|
|
}
|
|
|
|
async function expectLocalRacePageLoaded(page) {
|
|
await page.waitForSelector('form[onsubmit="return searching()"]', { state: 'visible' });
|
|
const raceId = await page.locator('#id_gara').inputValue();
|
|
if (!/\d+/u.test(raceId)) {
|
|
throw new Error(`Expected the local race page to expose a numeric race id, got: ${raceId}`);
|
|
}
|
|
|
|
await page.waitForSelector('#faceaiLaunchButton', { state: 'visible' });
|
|
await page.waitForFunction(() => {
|
|
return document.querySelectorAll('a[data-faceai-photo-id] img.thumb').length > 0;
|
|
}, null, {
|
|
timeout: 30 * 1000
|
|
});
|
|
}
|
|
|
|
async function ensureLocalAuthenticatedRacePage(page, options = {}) {
|
|
await page.goto(LEGACY_LOGIN_URL, { waitUntil: 'domcontentloaded' });
|
|
await page.locator('#login').fill(LEGACY_USERNAME);
|
|
await page.locator('#pwd').fill(LEGACY_PASSWORD);
|
|
|
|
const submitLocator = page.locator('input[type="submit"], button[type="submit"], a.btn').filter({ hasText: /Accedi|Sign in/i }).first();
|
|
if (await submitLocator.count()) {
|
|
await Promise.all([
|
|
page.waitForNavigation({ waitUntil: 'domcontentloaded' }),
|
|
submitLocator.click()
|
|
]);
|
|
} else {
|
|
await Promise.all([
|
|
page.waitForNavigation({ waitUntil: 'domcontentloaded' }),
|
|
page.evaluate(() => {
|
|
const form = document.querySelector('form[action="Logon.abl"]');
|
|
if (!form) {
|
|
throw new Error('Local login page did not expose a Logon.abl form.');
|
|
}
|
|
|
|
const cmdIUField = form.querySelector('[name="cmdIU"]');
|
|
if (cmdIUField) {
|
|
cmdIUField.value = 'check';
|
|
}
|
|
form.submit();
|
|
})
|
|
]);
|
|
}
|
|
|
|
await page.goto(buildSimulatorUrl(options), { waitUntil: 'domcontentloaded' });
|
|
await expectLocalRacePageLoaded(page);
|
|
}
|
|
|
|
function getSearchArtifacts(searchId) {
|
|
const searchRoot = path.join(SEARCH_LOG_ROOT, searchId);
|
|
return {
|
|
searchRoot,
|
|
backendLogPath: path.join(LOG_ROOT, 'backend.log'),
|
|
processorLogPath: path.join(LOG_ROOT, 'processor.log'),
|
|
auditDbPath: fsSync.existsSync(AUDIT_DB_PATH) ? AUDIT_DB_PATH : `${AUDIT_DB_QUERY_CONTAINER}:${AUDIT_DB_PATH_IN_CONTAINER}`,
|
|
workerLogPath: path.join(searchRoot, 'worker.log'),
|
|
matcherLogPath: path.join(searchRoot, 'matcher.log')
|
|
};
|
|
}
|
|
|
|
function parseAuditArtifacts(result) {
|
|
return {
|
|
searchRow: result.searchRow ? {
|
|
...result.searchRow,
|
|
matches: result.searchRow.matches_json ? JSON.parse(result.searchRow.matches_json) : null
|
|
} : null,
|
|
events: result.events.map((event) => ({
|
|
...event,
|
|
payload: event.payload_json ? JSON.parse(event.payload_json) : null
|
|
})),
|
|
fingerprintMatches: result.fingerprintMatches || []
|
|
};
|
|
}
|
|
|
|
function readAuditArtifactsFromContainer(searchId) {
|
|
const script = `
|
|
const { DatabaseSync } = require('node:sqlite');
|
|
const db = new DatabaseSync(${JSON.stringify(AUDIT_DB_PATH_IN_CONTAINER)}, { readOnly: true });
|
|
|
|
try {
|
|
const searchId = ${JSON.stringify(String(searchId))};
|
|
const searchRow = db.prepare(\`
|
|
SELECT
|
|
search_id,
|
|
requested_at,
|
|
completed_at,
|
|
redirect_issued_at,
|
|
status,
|
|
completion_code,
|
|
error_code,
|
|
error_message,
|
|
user_id,
|
|
user_display_name,
|
|
user_membership_status,
|
|
race_id,
|
|
race_name,
|
|
race_storage,
|
|
lang,
|
|
request_ip,
|
|
request_user_agent,
|
|
return_url,
|
|
selfie_name,
|
|
selfie_sha256,
|
|
selfie_size_bytes,
|
|
upload_path,
|
|
result_id,
|
|
match_count,
|
|
matches_json
|
|
FROM faceai_audit_searches
|
|
WHERE search_id = ?
|
|
\`).get(searchId);
|
|
|
|
const events = db.prepare(\`
|
|
SELECT
|
|
event_type,
|
|
happened_at,
|
|
status,
|
|
user_id,
|
|
race_id,
|
|
selfie_sha256,
|
|
result_id,
|
|
match_count,
|
|
completion_code,
|
|
error_code,
|
|
payload_json
|
|
FROM faceai_audit_events
|
|
WHERE search_id = ?
|
|
ORDER BY happened_at ASC, id ASC
|
|
\`).all(searchId);
|
|
|
|
const fingerprintMatches = searchRow?.selfie_sha256
|
|
? db.prepare(\`
|
|
SELECT search_id, result_id, match_count, status
|
|
FROM faceai_audit_searches
|
|
WHERE selfie_sha256 = ?
|
|
ORDER BY requested_at DESC
|
|
\`).all(searchRow.selfie_sha256)
|
|
: [];
|
|
|
|
console.log(JSON.stringify({ searchRow, events, fingerprintMatches }));
|
|
} finally {
|
|
db.close();
|
|
}
|
|
`;
|
|
|
|
const result = spawnSync('docker', ['exec', AUDIT_DB_QUERY_CONTAINER, 'node', '-e', script], {
|
|
cwd: ROOT_DIR,
|
|
encoding: 'utf8'
|
|
});
|
|
|
|
if (result.status !== 0) {
|
|
const details = (result.stderr || result.stdout || '').trim();
|
|
throw new Error(`Failed to read audit DB from ${AUDIT_DB_QUERY_CONTAINER}:${AUDIT_DB_PATH_IN_CONTAINER}${details ? `\n${details}` : ''}`);
|
|
}
|
|
|
|
return parseAuditArtifacts(JSON.parse(result.stdout));
|
|
}
|
|
|
|
function readAuditArtifacts(searchId) {
|
|
if (PREFER_CONTAINER_AUDIT_DB || !fsSync.existsSync(AUDIT_DB_PATH)) {
|
|
return readAuditArtifactsFromContainer(searchId);
|
|
}
|
|
|
|
const db = new DatabaseSync(AUDIT_DB_PATH, { readOnly: true });
|
|
|
|
try {
|
|
const searchRow = db.prepare(`
|
|
SELECT
|
|
search_id,
|
|
requested_at,
|
|
completed_at,
|
|
redirect_issued_at,
|
|
status,
|
|
completion_code,
|
|
error_code,
|
|
error_message,
|
|
user_id,
|
|
user_display_name,
|
|
user_membership_status,
|
|
race_id,
|
|
race_name,
|
|
race_storage,
|
|
lang,
|
|
request_ip,
|
|
request_user_agent,
|
|
return_url,
|
|
selfie_name,
|
|
selfie_sha256,
|
|
selfie_size_bytes,
|
|
upload_path,
|
|
result_id,
|
|
match_count,
|
|
matches_json
|
|
FROM faceai_audit_searches
|
|
WHERE search_id = ?
|
|
`).get(String(searchId));
|
|
|
|
const events = db.prepare(`
|
|
SELECT
|
|
event_type,
|
|
happened_at,
|
|
status,
|
|
user_id,
|
|
race_id,
|
|
selfie_sha256,
|
|
result_id,
|
|
match_count,
|
|
completion_code,
|
|
error_code,
|
|
payload_json
|
|
FROM faceai_audit_events
|
|
WHERE search_id = ?
|
|
ORDER BY happened_at ASC, id ASC
|
|
`).all(String(searchId));
|
|
|
|
const fingerprintMatches = searchRow?.selfie_sha256
|
|
? db.prepare(`
|
|
SELECT search_id, result_id, match_count, status
|
|
FROM faceai_audit_searches
|
|
WHERE selfie_sha256 = ?
|
|
ORDER BY requested_at DESC
|
|
`).all(searchRow.selfie_sha256)
|
|
: [];
|
|
|
|
return parseAuditArtifacts({ searchRow, events, fingerprintMatches });
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
async function readUtf8(filePath) {
|
|
return fs.readFile(filePath, 'utf8');
|
|
}
|
|
|
|
module.exports = {
|
|
ROOT_DIR,
|
|
LOG_ROOT,
|
|
SEARCH_LOG_ROOT,
|
|
AUDIT_DB_PATH,
|
|
FACEAI_BASE_URL,
|
|
LEGACY_BASE_URL,
|
|
LEGACY_HOME_URL,
|
|
LEGACY_LOGIN_URL,
|
|
LEGACY_PASSWORD,
|
|
SIMULATOR_URL,
|
|
SELFIE_NAME,
|
|
EXPECTED_MATCH_COUNT,
|
|
LEGACY_RACE_ID,
|
|
LEGACY_USERNAME,
|
|
buildHandoffUrl,
|
|
buildLegacyLoginFormData,
|
|
buildSimulatorUrl,
|
|
dockerCompose,
|
|
ensureLocalAuthenticatedRacePage,
|
|
expectLocalRacePageLoaded,
|
|
getSearchArtifacts,
|
|
readAuditArtifacts,
|
|
getSelfiePath,
|
|
performLocalLoginRequest,
|
|
prepareHostState,
|
|
readUtf8,
|
|
runCommand,
|
|
waitForHttp
|
|
}; |