feat: add processor service with Redis-backed job queue

- Introduced a new `processor` service in the Docker Compose setup to handle face matching jobs.
- Configured Redis as a job queue and state management system for processing searches.
- Updated the backend to enqueue jobs and manage user locks using Redis.
- Added environment variables for Redis configuration and runtime paths.
- Created technical design documentation for the processor service outlining architecture, queue model, and search lifecycle.
- Updated package.json and package-lock.json to include dependencies for BullMQ and ioredis in the processor workspace.
- Added sample PKL files for local testing in the `test_pkl` directory.

faceai/apps/backend/src/redis-store.js

@@ -0,0 +1,138 @@
+import Redis from 'ioredis';
+import { randomId } from './auth.js';
+
+export function createRedisConnection(redisUrl) {
+  return new Redis(redisUrl, {
+    maxRetriesPerRequest: null,
+    enableReadyCheck: true
+  });
+}
+
+function searchKey(searchId) {
+  return `faceai:search:${searchId}`;
+}
+
+function resultKey(resultId) {
+  return `faceai:result:${resultId}`;
+}
+
+function activeSearchKey(userId) {
+  return `faceai:active-search:user:${userId}`;
+}
+
+function rateLimitKey(userId) {
+  return `faceai:rate-limit:${userId}`;
+}
+
+export async function incrementRateLimit(redis, userId, windowSeconds) {
+  const key = rateLimitKey(userId);
+  const count = await redis.incr(key);
+  if (count === 1) {
+    await redis.expire(key, windowSeconds);
+  }
+  return count;
+}
+
+export async function acquireActiveSearchLock(redis, userId, searchId, ttlSeconds) {
+  const result = await redis.set(activeSearchKey(userId), searchId, 'EX', ttlSeconds, 'NX');
+  return result === 'OK';
+}
+
+export async function releaseActiveSearchLock(redis, userId, searchId) {
+  const key = activeSearchKey(userId);
+  const current = await redis.get(key);
+  if (current === String(searchId)) {
+    await redis.del(key);
+  }
+}
+
+export async function getActiveSearchId(redis, userId) {
+  return redis.get(activeSearchKey(userId));
+}
+
+export async function createSearchRecord(redis, payload, ttlSeconds) {
+  const searchId = randomId('search');
+  const record = {
+    id: searchId,
+    status: 'queued',
+    resultId: null,
+    matchCount: 0,
+    errorCode: null,
+    errorMessage: null,
+    createdAt: Date.now(),
+    startedAt: null,
+    completedAt: null,
+    ...payload
+  };
+
+  await redis.set(searchKey(searchId), JSON.stringify(record), 'EX', ttlSeconds);
+  return record;
+}
+
+export async function saveSearchRecord(redis, record, ttlSeconds) {
+  await redis.set(searchKey(record.id), JSON.stringify(record), 'EX', ttlSeconds);
+  return record;
+}
+
+export async function getSearchRecord(redis, searchId) {
+  const raw = await redis.get(searchKey(searchId));
+  return raw ? JSON.parse(raw) : null;
+}
+
+async function updateSearchRecord(redis, searchId, updater, ttlSeconds) {
+  const current = await getSearchRecord(redis, searchId);
+  if (!current) {
+    return null;
+  }
+
+  const next = updater(current);
+  await redis.set(searchKey(searchId), JSON.stringify(next), 'EX', ttlSeconds);
+  return next;
+}
+
+export async function markSearchProcessing(redis, searchId, ttlSeconds = 24 * 60 * 60) {
+  return updateSearchRecord(redis, searchId, (current) => ({
+    ...current,
+    status: 'processing',
+    startedAt: Date.now(),
+    errorCode: null,
+    errorMessage: null
+  }), ttlSeconds);
+}
+
+export async function markSearchCompleted(redis, searchId, resultId, matchCount, ttlSeconds) {
+  return updateSearchRecord(redis, searchId, (current) => ({
+    ...current,
+    status: 'completed',
+    resultId,
+    matchCount,
+    completedAt: Date.now()
+  }), ttlSeconds);
+}
+
+export async function markSearchFailed(redis, searchId, errorCode, errorMessage, ttlSeconds) {
+  return updateSearchRecord(redis, searchId, (current) => ({
+    ...current,
+    status: 'failed',
+    errorCode,
+    errorMessage,
+    completedAt: Date.now()
+  }), ttlSeconds);
+}
+
+export async function storeResultRecord(redis, payload, ttlSeconds) {
+  const resultId = randomId('result');
+  const record = {
+    id: resultId,
+    createdAt: Date.now(),
+    ...payload
+  };
+
+  await redis.set(resultKey(resultId), JSON.stringify(record), 'EX', ttlSeconds);
+  return record;
+}
+
+export async function getResultRecord(redis, resultId) {
+  const raw = await redis.get(resultKey(resultId));
+  return raw ? JSON.parse(raw) : null;
+}