Hot reload proactif post-cron pour ratings + mappings + chunks (evite la latence du 1er appel)

lib/dataReload.js

@@ -0,0 +1,81 @@
+// Centralised hot-reload watcher.
+//
+// After the cron rewrites the in-memory data files (search chunks, IMDb
+// mappings, IMDb ratings TSV), the existing per-call mtime checks already
+// reload them lazily — but the FIRST request post-cron pays a 1-2 s penalty.
+//
+// This module watches all four data sources and proactively triggers a reload
+// in the background as soon as the files change, so users never hit the cold
+// path. Debounced because the cron writes ~10 files within seconds.
+
+import { unwatchFile, watch, watchFile } from 'node:fs';
+import { IMDB_RATINGS, TMDBINTEGRAL_DIR } from '../config.js';
+import { preloadMappings } from './imdbMapping.js';
+import { getRatings } from './imdbRatings.js';
+import { imdbRatingsCount } from './metrics.js';
+import { reloadAllPools } from './searchEngine.js';
+
+const DEBOUNCE_MS = 5000;
+const timers = { chunks: null, mappings: null, ratings: null };
+const watchers = [];
+
+function debounce(key, fn) {
+  clearTimeout(timers[key]);
+  timers[key] = setTimeout(() => {
+    Promise.resolve(fn()).catch((err) => console.error(`Hot reload (${key}) failed:`, err.message));
+  }, DEBOUNCE_MS);
+}
+
+async function reloadRatings() {
+  // getRatings() checks the file mtime and rebuilds the Map if it changed.
+  // Calling it here pre-populates the cache so the next request is hot.
+  const map = await getRatings();
+  imdbRatingsCount.set(map.size);
+  console.log(`Hot reload: imdbratings.tsv (${map.size} entries)`);
+}
+
+async function reloadMappings() {
+  const out = await preloadMappings();
+  console.log(`Hot reload: IMDb mappings (movie=${out.movie}, tv=${out.tv})`);
+}
+
+async function reloadChunks() {
+  await reloadAllPools();
+}
+
+export function startWatchers() {
+  // tmdbintegral/ — covers search chunks AND imdb2{movie,tv}.json
+  try {
+    const w = watch(TMDBINTEGRAL_DIR, (_event, filename) => {
+      if (!filename) return;
+      if (/^search(movie|tv)\d+\.json$/.test(filename)) {
+        debounce('chunks', reloadChunks);
+      } else if (filename === 'imdb2movie.json' || filename === 'imdb2tv.json') {
+        debounce('mappings', reloadMappings);
+      }
+    });
+    w.unref();
+    watchers.push(w);
+  } catch (err) {
+    console.warn(`Cannot watch ${TMDBINTEGRAL_DIR}:`, err.message);
+  }
+
+  // imdbratings.tsv — fs.watch on a single file across a rename(tmp -> file)
+  // is unreliable on Linux because the inode changes. fs.watchFile (poll
+  // every 10 s) follows the path, not the inode. Cost: one stat() every 10 s.
+  watchFile(IMDB_RATINGS, { interval: 10_000 }, (curr, prev) => {
+    if (curr.mtimeMs !== prev.mtimeMs) debounce('ratings', reloadRatings);
+  });
+  watchers.push({ close: () => unwatchFile(IMDB_RATINGS) });
+}
+
+export function stopWatchers() {
+  for (const w of watchers) {
+    try {
+      w.close();
+    } catch {
+      /* ignore */
+    }
+  }
+  watchers.length = 0;
+}

lib/searchEngine.js

@@ -3,10 +3,10 @@
 // stay loaded in memory (replaces the per-request `php searchmultithreads.php`
 // fork from the PHP version).
 //
-// A filesystem watcher detects when the cron rewrites the chunks and recycles
+// Hot reload of chunks after a cron rewrite is handled by lib/dataReload.js
-// the worker pool transparently — no server restart needed.
+// which calls reloadAllPools() exported below.
 
-import { existsSync, watch } from 'node:fs';
+import { existsSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { Worker } from 'node:worker_threads';
@@ -16,9 +16,6 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 const WORKER_PATH = join(__dirname, 'searchWorker.js');
 
 const pools = new Map();
-let watcher = null;
-let reloadTimer = null;
-const RELOAD_DEBOUNCE_MS = 5000;
 
 class WorkerPool {
   constructor(type) {
@@ -67,8 +64,9 @@ class WorkerPool {
   }
 }
 
-async function reloadAllPools() {
+export async function reloadAllPools() {
   const types = [...pools.keys()];
+  if (!types.length) return;
   console.log(`Reloading search pools: ${types.join(', ')}`);
   for (const type of types) {
     const old = pools.get(type);
@@ -79,25 +77,9 @@ async function reloadAllPools() {
   }
 }
 
-function ensureWatcher() {
-  if (watcher) return;
-  try {
-    watcher = watch(TMDBINTEGRAL_DIR, (_event, filename) => {
-      if (!filename) return;
-      if (!/^search(movie|tv)\d+\.json$/.test(filename)) return;
-      clearTimeout(reloadTimer);
-      reloadTimer = setTimeout(reloadAllPools, RELOAD_DEBOUNCE_MS);
-    });
-    watcher.unref();
-  } catch (err) {
-    console.warn(`Cannot watch ${TMDBINTEGRAL_DIR} for chunk reload:`, err.message);
-  }
-}
-
 export function getPool(type) {
   if (!pools.has(type)) {
     pools.set(type, new WorkerPool(type));
-    ensureWatcher();
   }
   return pools.get(type);
 }

server.js

@@ -5,6 +5,7 @@ import secureSession from '@fastify/secure-session';
 import fastifyStatic from '@fastify/static';
 import Fastify from 'fastify';
 import { HOST, PORT, RATE_LIMIT_PER_SEC, ROOT, SESSION_SECRET } from './config.js';
+import { startWatchers } from './lib/dataReload.js';
 import { preloadMappings } from './lib/imdbMapping.js';
 import { getRatings } from './lib/imdbRatings.js';
 import { httpDuration, httpRequests, imdbRatingsCount, searchWorkers } from './lib/metrics.js';
@@ -82,9 +83,10 @@ fastify.ready().then(async () => {
     searchWorkers.set({ type: 'movie' }, movie.workers.length);
     searchWorkers.set({ type: 'tv' }, tv.workers.length);
     const maps = await preloadMappings();
+    startWatchers();
     fastify.log.info(
       { ratings: ratings.size, imdb_movie: maps.movie, imdb_tv: maps.tv },
-      'Warmup complete',
+      'Warmup complete (data hot-reload watchers started)',
     );
   } catch (err) {
     fastify.log.warn({ err }, 'Warmup failed');