feat(audit): implement audit logging for search requests and results
All checks were successful
Publish FaceAI Container / publish (push) Successful in 13m22s
All checks were successful
Publish FaceAI Container / publish (push) Successful in 13m22s
- Added configuration options for audit database path and retention days in backend and processor. - Integrated audit logging in server and worker processes to track search requests, completions, and failures. - Created utility functions for reading and parsing audit logs in end-to-end tests. - Updated Docker Compose files to include audit database configuration. - Added new tests to verify audit log entries for successful and no-results searches.
This commit is contained in:
parent
a026fec62b
commit
32db61c381
14 changed files with 1067 additions and 16 deletions
|
|
@ -3,26 +3,35 @@ const {
|
|||
ensureLocalAuthenticatedRacePage,
|
||||
EXPECTED_MATCH_COUNT,
|
||||
FACEAI_BASE_URL,
|
||||
LEGACY_BASE_URL,
|
||||
LEGACY_RACE_ID,
|
||||
SELFIE_NAME,
|
||||
buildHandoffUrl,
|
||||
buildSimulatorUrl,
|
||||
getSearchArtifacts,
|
||||
getSelfiePath,
|
||||
readAuditArtifacts,
|
||||
readUtf8
|
||||
} = require('./faceai-test-utils');
|
||||
|
||||
const FACEAI_HOME_URL_RE = /http:\/\/(localhost|127\.0\.0\.1):3001\/?(?:\?.*)?$/;
|
||||
const FACEAI_CALLBACK_URL_RE = /http:\/\/(localhost|127\.0\.0\.1):3001\/auth\/callback\?token=/;
|
||||
const LEGACY_HOME_URL_RE = /http:\/\/(localhost|127\.0\.0\.1):8080\/index\.jsp$/;
|
||||
const LONG_TEST_TIMEOUT_MS = 3 * 60 * 1000;
|
||||
const SHORT_UI_TIMEOUT_MS = 30 * 1000;
|
||||
const SEARCH_COMPLETION_TIMEOUT_MS = 75 * 1000;
|
||||
const LEGACY_RETURN_TIMEOUT_MS = 75 * 1000;
|
||||
const FILE_CHOOSER_TIMEOUT_MS = 8 * 1000;
|
||||
const FACEAI_CONSENT_HEADING_RE = /Prima di continuare|Before you continue/i;
|
||||
const FACEAI_UPLOAD_HEADING_RE = /Carica il tuo selfie|Upload your selfie/i;
|
||||
const LEGACY_BASE_URL_RE = new RegExp(`^${escapeRegExp(LEGACY_BASE_URL)}`);
|
||||
const LEGACY_HOME_URL_RE = new RegExp(`^${escapeRegExp(`${LEGACY_BASE_URL}/index.jsp`)}$`);
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
function buildLegacySimulatorReturnMatcher(raceId) {
|
||||
return new RegExp(`http://(localhost|127\\.0\\.0\\.1):8080/Foto2\\.abl\\?id_gara=${raceId}.*`);
|
||||
return new RegExp(`^${escapeRegExp(`${LEGACY_BASE_URL}/Foto2.abl?id_gara=${raceId}`)}.*`);
|
||||
}
|
||||
|
||||
function assertLogDoesNotContain(content, patterns, label) {
|
||||
|
|
@ -32,10 +41,20 @@ function assertLogDoesNotContain(content, patterns, label) {
|
|||
}
|
||||
|
||||
async function waitForFaceAiHome(page) {
|
||||
await page.waitForURL((url) => FACEAI_CALLBACK_URL_RE.test(url.toString()) || FACEAI_HOME_URL_RE.test(url.toString()), {
|
||||
await page.waitForURL((url) => FACEAI_HOME_URL_RE.test(url.toString()), {
|
||||
timeout: SHORT_UI_TIMEOUT_MS
|
||||
});
|
||||
await expect(page.getByRole('heading', { name: 'Trova le tue foto con un selfie' })).toBeVisible();
|
||||
|
||||
const consentHeading = page.getByRole('heading', { name: FACEAI_CONSENT_HEADING_RE });
|
||||
if (await consentHeading.isVisible().catch(() => false)) {
|
||||
await page.getByRole('checkbox', {
|
||||
name: /Confermo di aver letto l’informativa sul trattamento dei dati biometrici\.|I confirm that I have read the biometric data processing notice\./i
|
||||
}).check();
|
||||
await page.getByRole('button', { name: /Accetto e continuo|I agree and continue/i }).click();
|
||||
}
|
||||
|
||||
await expect(page.getByRole('heading', { name: FACEAI_UPLOAD_HEADING_RE })).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: /Scegli immagine|Choose image/i })).toBeVisible();
|
||||
}
|
||||
|
||||
async function launchFromSimulator(page, options = {}) {
|
||||
|
|
@ -59,11 +78,10 @@ async function readLaunchUrlFromLegacyPage(page) {
|
|||
});
|
||||
|
||||
expect(launchUrl, 'Expected the legacy race page to expose a FaceAI handoff URL builder.').toBeTruthy();
|
||||
return new URL(launchUrl, 'http://127.0.0.1:8080');
|
||||
return new URL(launchUrl, LEGACY_BASE_URL);
|
||||
}
|
||||
|
||||
async function startSearch(page, selfieName) {
|
||||
const selfieLabel = selfieName.split(/[\\/]+/u).pop();
|
||||
const createResponsePromise = page.waitForResponse((response) => {
|
||||
return response.url().includes('/api/searches')
|
||||
&& response.request().method() === 'POST'
|
||||
|
|
@ -71,8 +89,6 @@ async function startSearch(page, selfieName) {
|
|||
});
|
||||
|
||||
await page.locator('input[type="file"]').setInputFiles(getSelfiePath(selfieName));
|
||||
await expect(page.getByText(selfieLabel)).toBeVisible();
|
||||
await page.getByRole('button', { name: 'Avvia ricerca Face ID' }).click();
|
||||
|
||||
const createResponse = await createResponsePromise;
|
||||
return createResponse.json();
|
||||
|
|
@ -151,6 +167,79 @@ async function verifySearchLogs(searchId, { expectedMatchCount, expectedSelfieNa
|
|||
return { backendLog, processorLog, workerLog, matcherLog };
|
||||
}
|
||||
|
||||
async function verifyAuditLog(searchId, { expectedMatchCount, expectedRaceId, expectedSelfieName, expectedUserId }) {
|
||||
const artifacts = getSearchArtifacts(searchId);
|
||||
const audit = readAuditArtifacts(searchId);
|
||||
|
||||
expect(audit.searchRow, `Expected ${artifacts.auditDbPath} to contain an audit row for ${searchId}`).toBeTruthy();
|
||||
expect(audit.searchRow.search_id).toBe(searchId);
|
||||
expect(audit.searchRow.status).toBe('completed');
|
||||
expect(audit.searchRow.match_count).toBe(expectedMatchCount);
|
||||
expect(audit.searchRow.race_id).toBe(expectedRaceId);
|
||||
expect(audit.searchRow.user_id).toBe(expectedUserId);
|
||||
expect(audit.searchRow.selfie_name).toBe(expectedSelfieName);
|
||||
expect(audit.searchRow.selfie_sha256).toMatch(/^[a-f0-9]{64}$/i);
|
||||
expect(audit.searchRow.selfie_size_bytes).toBeGreaterThan(0);
|
||||
expect(audit.searchRow.result_id).toBeTruthy();
|
||||
expect(audit.searchRow.requested_at).toBeGreaterThan(0);
|
||||
expect(audit.searchRow.completed_at).toBeGreaterThan(0);
|
||||
expect(audit.searchRow.redirect_issued_at).toBeGreaterThan(0);
|
||||
expect(audit.searchRow.matches).toHaveLength(expectedMatchCount);
|
||||
|
||||
const eventTypes = audit.events.map((event) => event.event_type);
|
||||
expect(eventTypes).toContain('search_requested');
|
||||
expect(eventTypes).toContain('search_completed');
|
||||
expect(eventTypes).toContain('search_redirect_issued');
|
||||
|
||||
const requestedEvent = audit.events.find((event) => event.event_type === 'search_requested');
|
||||
expect(requestedEvent?.selfie_sha256).toBe(audit.searchRow.selfie_sha256);
|
||||
|
||||
const fingerprintMatch = audit.fingerprintMatches.find((entry) => entry.search_id === searchId);
|
||||
expect(fingerprintMatch, 'Expected fingerprint lookup to find the original search row').toBeTruthy();
|
||||
expect(fingerprintMatch.match_count).toBe(expectedMatchCount);
|
||||
expect(fingerprintMatch.result_id).toBe(audit.searchRow.result_id);
|
||||
expect(fingerprintMatch.status).toBe('completed');
|
||||
|
||||
return audit;
|
||||
}
|
||||
|
||||
async function verifyNoResultsAuditLog(searchId, { expectedRaceId, expectedSelfieName, expectedUserId }) {
|
||||
const artifacts = getSearchArtifacts(searchId);
|
||||
const audit = readAuditArtifacts(searchId);
|
||||
|
||||
expect(audit.searchRow, `Expected ${artifacts.auditDbPath} to contain an audit row for ${searchId}`).toBeTruthy();
|
||||
expect(audit.searchRow.search_id).toBe(searchId);
|
||||
expect(audit.searchRow.status).toBe('completed');
|
||||
expect(audit.searchRow.completion_code).toBe('NO_FACES_FOUND');
|
||||
expect(audit.searchRow.match_count).toBe(0);
|
||||
expect(audit.searchRow.race_id).toBe(expectedRaceId);
|
||||
expect(audit.searchRow.user_id).toBe(expectedUserId);
|
||||
expect(audit.searchRow.selfie_name).toBe(expectedSelfieName);
|
||||
expect(audit.searchRow.selfie_sha256).toMatch(/^[a-f0-9]{64}$/i);
|
||||
expect(audit.searchRow.selfie_size_bytes).toBeGreaterThan(0);
|
||||
expect(audit.searchRow.result_id).toBeTruthy();
|
||||
expect(audit.searchRow.requested_at).toBeGreaterThan(0);
|
||||
expect(audit.searchRow.completed_at).toBeGreaterThan(0);
|
||||
expect(audit.searchRow.redirect_issued_at).toBeNull();
|
||||
expect(audit.searchRow.matches).toHaveLength(0);
|
||||
|
||||
const eventTypes = audit.events.map((event) => event.event_type);
|
||||
expect(eventTypes).toContain('search_requested');
|
||||
expect(eventTypes).toContain('search_completed');
|
||||
expect(eventTypes).not.toContain('search_redirect_issued');
|
||||
|
||||
const requestedEvent = audit.events.find((event) => event.event_type === 'search_requested');
|
||||
expect(requestedEvent?.selfie_sha256).toBe(audit.searchRow.selfie_sha256);
|
||||
|
||||
const fingerprintMatch = audit.fingerprintMatches.find((entry) => entry.search_id === searchId);
|
||||
expect(fingerprintMatch, 'Expected fingerprint lookup to find the original search row').toBeTruthy();
|
||||
expect(fingerprintMatch.match_count).toBe(0);
|
||||
expect(fingerprintMatch.result_id).toBe(audit.searchRow.result_id);
|
||||
expect(fingerprintMatch.status).toBe('completed');
|
||||
|
||||
return audit;
|
||||
}
|
||||
|
||||
async function closeContexts(contexts) {
|
||||
await Promise.all(contexts.map(async (context) => {
|
||||
try {
|
||||
|
|
@ -181,6 +270,41 @@ test('runs the legacy Tomcat flow through FaceAI and returns to the filtered leg
|
|||
expectedMatchCount: EXPECTED_MATCH_COUNT,
|
||||
expectedSelfieName: SELFIE_NAME.split(/[\\/]+/u).pop()
|
||||
});
|
||||
|
||||
await verifyAuditLog(search.id, {
|
||||
expectedMatchCount: EXPECTED_MATCH_COUNT,
|
||||
expectedRaceId: LEGACY_RACE_ID,
|
||||
expectedSelfieName: SELFIE_NAME.split(/[\\/]+/u).pop(),
|
||||
expectedUserId: 'legacy-user-1'
|
||||
});
|
||||
});
|
||||
|
||||
test('records structured logs for a completed no-results FaceAI search in the dev compose stack', async ({ page }) => {
|
||||
test.slow();
|
||||
|
||||
await launchFromSimulator(page, {
|
||||
raceId: LEGACY_RACE_ID,
|
||||
raceSlug: 'isolotto',
|
||||
raceName: 'Festa sociale UP Isolotto',
|
||||
raceFolder: 'ISOLOTTO'
|
||||
});
|
||||
|
||||
const search = await startSearch(page, SELFIE_NAME);
|
||||
|
||||
await waitForSearchCondition(page, search.id, (payload) => {
|
||||
return payload.status === 'completed' && payload.completionCode === 'NO_FACES_FOUND';
|
||||
}, SEARCH_COMPLETION_TIMEOUT_MS);
|
||||
|
||||
await verifySearchLogs(search.id, {
|
||||
expectedMatchCount: 0,
|
||||
expectedSelfieName: SELFIE_NAME.split(/[\\/]+/u).pop()
|
||||
});
|
||||
|
||||
await verifyNoResultsAuditLog(search.id, {
|
||||
expectedRaceId: LEGACY_RACE_ID,
|
||||
expectedSelfieName: SELFIE_NAME.split(/[\\/]+/u).pop(),
|
||||
expectedUserId: 'legacy-user-1'
|
||||
});
|
||||
});
|
||||
|
||||
test('builds the legacy FaceAI handoff URL with the exact local race storage metadata', async ({ page }) => {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
const fs = require('node:fs/promises');
|
||||
const fsSync = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { spawn } = require('node:child_process');
|
||||
const { spawn, spawnSync } = require('node:child_process');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
|
||||
const ROOT_DIR = path.resolve(__dirname, '..', '..');
|
||||
const WORKSPACE_ROOT = path.resolve(ROOT_DIR, '..');
|
||||
const LOG_ROOT = path.join(ROOT_DIR, 'logs');
|
||||
const SEARCH_LOG_ROOT = path.join(LOG_ROOT, 'searches');
|
||||
const AUDIT_DB_PATH = path.join(LOG_ROOT, 'faceai-audit.sqlite');
|
||||
const PREFER_CONTAINER_AUDIT_DB = process.env.FACEAI_E2E_AUDIT_READ_FROM_CONTAINER === '1' || process.platform === 'win32';
|
||||
const AUDIT_DB_PATH_IN_CONTAINER = process.env.FACEAI_E2E_AUDIT_DB_PATH_IN_CONTAINER || '/data/runtime/faceai-audit.sqlite';
|
||||
const AUDIT_DB_QUERY_CONTAINER = process.env.FACEAI_E2E_AUDIT_QUERY_CONTAINER || 'regalami-faceai-processor';
|
||||
const FACEAI_BASE_URL = process.env.FACEAI_E2E_BASE_URL || 'http://127.0.0.1:3001';
|
||||
const SIMULATOR_URL = process.env.FACEAI_E2E_SIMULATOR_URL || 'http://127.0.0.1:8080/Foto2.abl?id_gara=1018547&pageRow=96&pageNumber=1';
|
||||
const LEGACY_BASE_URL = process.env.FACEAI_E2E_LEGACY_BASE_URL || 'http://127.0.0.1:8080';
|
||||
|
|
@ -260,11 +266,182 @@ function getSearchArtifacts(searchId) {
|
|||
searchRoot,
|
||||
backendLogPath: path.join(LOG_ROOT, 'backend.log'),
|
||||
processorLogPath: path.join(LOG_ROOT, 'processor.log'),
|
||||
auditDbPath: fsSync.existsSync(AUDIT_DB_PATH) ? AUDIT_DB_PATH : `${AUDIT_DB_QUERY_CONTAINER}:${AUDIT_DB_PATH_IN_CONTAINER}`,
|
||||
workerLogPath: path.join(searchRoot, 'worker.log'),
|
||||
matcherLogPath: path.join(searchRoot, 'matcher.log')
|
||||
};
|
||||
}
|
||||
|
||||
function parseAuditArtifacts(result) {
|
||||
return {
|
||||
searchRow: result.searchRow ? {
|
||||
...result.searchRow,
|
||||
matches: result.searchRow.matches_json ? JSON.parse(result.searchRow.matches_json) : null
|
||||
} : null,
|
||||
events: result.events.map((event) => ({
|
||||
...event,
|
||||
payload: event.payload_json ? JSON.parse(event.payload_json) : null
|
||||
})),
|
||||
fingerprintMatches: result.fingerprintMatches || []
|
||||
};
|
||||
}
|
||||
|
||||
function readAuditArtifactsFromContainer(searchId) {
|
||||
const script = `
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const db = new DatabaseSync(${JSON.stringify(AUDIT_DB_PATH_IN_CONTAINER)}, { readOnly: true });
|
||||
|
||||
try {
|
||||
const searchId = ${JSON.stringify(String(searchId))};
|
||||
const searchRow = db.prepare(\`
|
||||
SELECT
|
||||
search_id,
|
||||
requested_at,
|
||||
completed_at,
|
||||
redirect_issued_at,
|
||||
status,
|
||||
completion_code,
|
||||
error_code,
|
||||
error_message,
|
||||
user_id,
|
||||
user_display_name,
|
||||
user_membership_status,
|
||||
race_id,
|
||||
race_name,
|
||||
race_storage,
|
||||
lang,
|
||||
request_ip,
|
||||
request_user_agent,
|
||||
return_url,
|
||||
selfie_name,
|
||||
selfie_sha256,
|
||||
selfie_size_bytes,
|
||||
upload_path,
|
||||
result_id,
|
||||
match_count,
|
||||
matches_json
|
||||
FROM faceai_audit_searches
|
||||
WHERE search_id = ?
|
||||
\`).get(searchId);
|
||||
|
||||
const events = db.prepare(\`
|
||||
SELECT
|
||||
event_type,
|
||||
happened_at,
|
||||
status,
|
||||
user_id,
|
||||
race_id,
|
||||
selfie_sha256,
|
||||
result_id,
|
||||
match_count,
|
||||
completion_code,
|
||||
error_code,
|
||||
payload_json
|
||||
FROM faceai_audit_events
|
||||
WHERE search_id = ?
|
||||
ORDER BY happened_at ASC, id ASC
|
||||
\`).all(searchId);
|
||||
|
||||
const fingerprintMatches = searchRow?.selfie_sha256
|
||||
? db.prepare(\`
|
||||
SELECT search_id, result_id, match_count, status
|
||||
FROM faceai_audit_searches
|
||||
WHERE selfie_sha256 = ?
|
||||
ORDER BY requested_at DESC
|
||||
\`).all(searchRow.selfie_sha256)
|
||||
: [];
|
||||
|
||||
console.log(JSON.stringify({ searchRow, events, fingerprintMatches }));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
`;
|
||||
|
||||
const result = spawnSync('docker', ['exec', AUDIT_DB_QUERY_CONTAINER, 'node', '-e', script], {
|
||||
cwd: ROOT_DIR,
|
||||
encoding: 'utf8'
|
||||
});
|
||||
|
||||
if (result.status !== 0) {
|
||||
const details = (result.stderr || result.stdout || '').trim();
|
||||
throw new Error(`Failed to read audit DB from ${AUDIT_DB_QUERY_CONTAINER}:${AUDIT_DB_PATH_IN_CONTAINER}${details ? `\n${details}` : ''}`);
|
||||
}
|
||||
|
||||
return parseAuditArtifacts(JSON.parse(result.stdout));
|
||||
}
|
||||
|
||||
function readAuditArtifacts(searchId) {
|
||||
if (PREFER_CONTAINER_AUDIT_DB || !fsSync.existsSync(AUDIT_DB_PATH)) {
|
||||
return readAuditArtifactsFromContainer(searchId);
|
||||
}
|
||||
|
||||
const db = new DatabaseSync(AUDIT_DB_PATH, { readOnly: true });
|
||||
|
||||
try {
|
||||
const searchRow = db.prepare(`
|
||||
SELECT
|
||||
search_id,
|
||||
requested_at,
|
||||
completed_at,
|
||||
redirect_issued_at,
|
||||
status,
|
||||
completion_code,
|
||||
error_code,
|
||||
error_message,
|
||||
user_id,
|
||||
user_display_name,
|
||||
user_membership_status,
|
||||
race_id,
|
||||
race_name,
|
||||
race_storage,
|
||||
lang,
|
||||
request_ip,
|
||||
request_user_agent,
|
||||
return_url,
|
||||
selfie_name,
|
||||
selfie_sha256,
|
||||
selfie_size_bytes,
|
||||
upload_path,
|
||||
result_id,
|
||||
match_count,
|
||||
matches_json
|
||||
FROM faceai_audit_searches
|
||||
WHERE search_id = ?
|
||||
`).get(String(searchId));
|
||||
|
||||
const events = db.prepare(`
|
||||
SELECT
|
||||
event_type,
|
||||
happened_at,
|
||||
status,
|
||||
user_id,
|
||||
race_id,
|
||||
selfie_sha256,
|
||||
result_id,
|
||||
match_count,
|
||||
completion_code,
|
||||
error_code,
|
||||
payload_json
|
||||
FROM faceai_audit_events
|
||||
WHERE search_id = ?
|
||||
ORDER BY happened_at ASC, id ASC
|
||||
`).all(String(searchId));
|
||||
|
||||
const fingerprintMatches = searchRow?.selfie_sha256
|
||||
? db.prepare(`
|
||||
SELECT search_id, result_id, match_count, status
|
||||
FROM faceai_audit_searches
|
||||
WHERE selfie_sha256 = ?
|
||||
ORDER BY requested_at DESC
|
||||
`).all(searchRow.selfie_sha256)
|
||||
: [];
|
||||
|
||||
return parseAuditArtifacts({ searchRow, events, fingerprintMatches });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function readUtf8(filePath) {
|
||||
return fs.readFile(filePath, 'utf8');
|
||||
}
|
||||
|
|
@ -273,6 +450,7 @@ module.exports = {
|
|||
ROOT_DIR,
|
||||
LOG_ROOT,
|
||||
SEARCH_LOG_ROOT,
|
||||
AUDIT_DB_PATH,
|
||||
FACEAI_BASE_URL,
|
||||
LEGACY_BASE_URL,
|
||||
LEGACY_HOME_URL,
|
||||
|
|
@ -290,6 +468,7 @@ module.exports = {
|
|||
ensureLocalAuthenticatedRacePage,
|
||||
expectLocalRacePageLoaded,
|
||||
getSearchArtifacts,
|
||||
readAuditArtifacts,
|
||||
getSelfiePath,
|
||||
performLocalLoginRequest,
|
||||
prepareHostState,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue