From 2e84e212eae6a8ef3e3155aaf9ece9cd75b122c0 Mon Sep 17 00:00:00 2001 From: unfr Date: Fri, 24 Apr 2026 07:58:22 +0200 Subject: [PATCH] Hot reload proactif post-cron pour ratings + mappings + chunks (evite la latence du 1er appel) --- lib/dataReload.js | 81 +++++++++++++++++++++++++++++++++++++++++++++ lib/searchEngine.js | 28 +++------------- server.js | 4 ++- 3 files changed, 89 insertions(+), 24 deletions(-) create mode 100644 lib/dataReload.js diff --git a/lib/dataReload.js b/lib/dataReload.js new file mode 100644 index 0000000..e4eba66 --- /dev/null +++ b/lib/dataReload.js @@ -0,0 +1,81 @@ +// Centralised hot-reload watcher. +// +// After the cron rewrites the in-memory data files (search chunks, IMDb +// mappings, IMDb ratings TSV), the existing per-call mtime checks already +// reload them lazily — but the FIRST request post-cron pays a 1-2 s penalty. +// +// This module watches all four data sources and proactively triggers a reload +// in the background as soon as the files change, so users never hit the cold +// path. Debounced because the cron writes ~10 files within seconds. + +import { unwatchFile, watch, watchFile } from 'node:fs'; +import { IMDB_RATINGS, TMDBINTEGRAL_DIR } from '../config.js'; +import { preloadMappings } from './imdbMapping.js'; +import { getRatings } from './imdbRatings.js'; +import { imdbRatingsCount } from './metrics.js'; +import { reloadAllPools } from './searchEngine.js'; + +const DEBOUNCE_MS = 5000; +const timers = { chunks: null, mappings: null, ratings: null }; +const watchers = []; + +function debounce(key, fn) { + clearTimeout(timers[key]); + timers[key] = setTimeout(() => { + Promise.resolve(fn()).catch((err) => console.error(`Hot reload (${key}) failed:`, err.message)); + }, DEBOUNCE_MS); +} + +async function reloadRatings() { + // getRatings() checks the file mtime and rebuilds the Map if it changed. + // Calling it here pre-populates the cache so the next request is hot. + const map = await getRatings(); + imdbRatingsCount.set(map.size); + console.log(`Hot reload: imdbratings.tsv (${map.size} entries)`); +} + +async function reloadMappings() { + const out = await preloadMappings(); + console.log(`Hot reload: IMDb mappings (movie=${out.movie}, tv=${out.tv})`); +} + +async function reloadChunks() { + await reloadAllPools(); +} + +export function startWatchers() { + // tmdbintegral/ — covers search chunks AND imdb2{movie,tv}.json + try { + const w = watch(TMDBINTEGRAL_DIR, (_event, filename) => { + if (!filename) return; + if (/^search(movie|tv)\d+\.json$/.test(filename)) { + debounce('chunks', reloadChunks); + } else if (filename === 'imdb2movie.json' || filename === 'imdb2tv.json') { + debounce('mappings', reloadMappings); + } + }); + w.unref(); + watchers.push(w); + } catch (err) { + console.warn(`Cannot watch ${TMDBINTEGRAL_DIR}:`, err.message); + } + + // imdbratings.tsv — fs.watch on a single file across a rename(tmp -> file) + // is unreliable on Linux because the inode changes. fs.watchFile (poll + // every 10 s) follows the path, not the inode. Cost: one stat() every 10 s. + watchFile(IMDB_RATINGS, { interval: 10_000 }, (curr, prev) => { + if (curr.mtimeMs !== prev.mtimeMs) debounce('ratings', reloadRatings); + }); + watchers.push({ close: () => unwatchFile(IMDB_RATINGS) }); +} + +export function stopWatchers() { + for (const w of watchers) { + try { + w.close(); + } catch { + /* ignore */ + } + } + watchers.length = 0; +} diff --git a/lib/searchEngine.js b/lib/searchEngine.js index f3e03ad..99da62d 100644 --- a/lib/searchEngine.js +++ b/lib/searchEngine.js @@ -3,10 +3,10 @@ // stay loaded in memory (replaces the per-request `php searchmultithreads.php` // fork from the PHP version). // -// A filesystem watcher detects when the cron rewrites the chunks and recycles -// the worker pool transparently — no server restart needed. +// Hot reload of chunks after a cron rewrite is handled by lib/dataReload.js +// which calls reloadAllPools() exported below. -import { existsSync, watch } from 'node:fs'; +import { existsSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { Worker } from 'node:worker_threads'; @@ -16,9 +16,6 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const WORKER_PATH = join(__dirname, 'searchWorker.js'); const pools = new Map(); -let watcher = null; -let reloadTimer = null; -const RELOAD_DEBOUNCE_MS = 5000; class WorkerPool { constructor(type) { @@ -67,8 +64,9 @@ class WorkerPool { } } -async function reloadAllPools() { +export async function reloadAllPools() { const types = [...pools.keys()]; + if (!types.length) return; console.log(`Reloading search pools: ${types.join(', ')}`); for (const type of types) { const old = pools.get(type); @@ -79,25 +77,9 @@ async function reloadAllPools() { } } -function ensureWatcher() { - if (watcher) return; - try { - watcher = watch(TMDBINTEGRAL_DIR, (_event, filename) => { - if (!filename) return; - if (!/^search(movie|tv)\d+\.json$/.test(filename)) return; - clearTimeout(reloadTimer); - reloadTimer = setTimeout(reloadAllPools, RELOAD_DEBOUNCE_MS); - }); - watcher.unref(); - } catch (err) { - console.warn(`Cannot watch ${TMDBINTEGRAL_DIR} for chunk reload:`, err.message); - } -} - export function getPool(type) { if (!pools.has(type)) { pools.set(type, new WorkerPool(type)); - ensureWatcher(); } return pools.get(type); } diff --git a/server.js b/server.js index 295b4ab..08ebdae 100644 --- a/server.js +++ b/server.js @@ -5,6 +5,7 @@ import secureSession from '@fastify/secure-session'; import fastifyStatic from '@fastify/static'; import Fastify from 'fastify'; import { HOST, PORT, RATE_LIMIT_PER_SEC, ROOT, SESSION_SECRET } from './config.js'; +import { startWatchers } from './lib/dataReload.js'; import { preloadMappings } from './lib/imdbMapping.js'; import { getRatings } from './lib/imdbRatings.js'; import { httpDuration, httpRequests, imdbRatingsCount, searchWorkers } from './lib/metrics.js'; @@ -82,9 +83,10 @@ fastify.ready().then(async () => { searchWorkers.set({ type: 'movie' }, movie.workers.length); searchWorkers.set({ type: 'tv' }, tv.workers.length); const maps = await preloadMappings(); + startWatchers(); fastify.log.info( { ratings: ratings.size, imdb_movie: maps.movie, imdb_tv: maps.tv }, - 'Warmup complete', + 'Warmup complete (data hot-reload watchers started)', ); } catch (err) { fastify.log.warn({ err }, 'Warmup failed');