From 54b6cc453e7dfc0d56b3fe181a810218ce9d7aac Mon Sep 17 00:00:00 2001 From: unfr Date: Fri, 24 Apr 2026 08:48:29 +0200 Subject: [PATCH] Dashboard admin : split stats/disk + preload disk au warmup + wc -l (1.85s -> 40ms) --- lib/stats.js | 71 ++++++++++++++++++++++----------------- public/admin.js | 88 +++++++++++++++++++++++++++---------------------- routes/admin.js | 7 +++- server.js | 6 +++- 4 files changed, 100 insertions(+), 72 deletions(-) diff --git a/lib/stats.js b/lib/stats.js index d910f20..ebbf0aa 100644 --- a/lib/stats.js +++ b/lib/stats.js @@ -45,10 +45,17 @@ async function cached(key, ttlMs, fn) { async function countLines(file) { if (!existsSync(file)) return 0; - let n = 0; - const rl = createInterface({ input: createReadStream(file), crlfDelay: Infinity }); - for await (const _ of rl) n++; - return n; + // wc -l is ~10x faster than Node readline on large files. Falls back to + // readline if wc is unavailable (non-Linux dev env). + try { + const { stdout } = await execFile('wc', ['-l', file], { maxBuffer: 1024 * 1024 }); + return parseInt(stdout.trim().split(/\s+/)[0], 10) || 0; + } catch { + let n = 0; + const rl = createInterface({ input: createReadStream(file), crlfDelay: Infinity }); + for await (const _ of rl) n++; + return n; + } } async function tailLines(file, n) { @@ -104,24 +111,11 @@ async function getCounterValue(name) { return arr.values.reduce((sum, v) => sum + (v.value || 0), 0); } -export async function getStats() { - const [ - movieTotal, - tvTotal, - cronText, - cronLogTail, - duMovie, - duTv, - duJwMovie, - duJwTv, - duRatings, - ratings, - mappings, - ] = await Promise.all([ - cached('movie_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'movie.json'))), - cached('tv_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'tv.json'))), - cached('cron_txt', 30_000, async () => (existsSync(CRON_TXT) ? await readFile(CRON_TXT, 'utf8') : '')), - cached('cron_tail', 30_000, () => tailLines(LASTCRON_TXT, 200)), +// Disk usage is the only slow part (du -sb on 1.7M files = ~2-3s cold, +// then 10 min cache). Exposed separately so the dashboard can show fast +// stats first and fill in disk asynchronously. +export async function getDiskUsage() { + const [duMovie, duTv, duJwMovie, duJwTv, duRatings] = await Promise.all([ cached('du_movie', 10 * 60_000, () => diskUsage(MOVIE_DIR)), cached('du_tv', 10 * 60_000, () => diskUsage(TV_DIR)), cached('du_jwmovie', 10 * 60_000, () => diskUsage(JUSTWATCH_MOVIE_DIR)), @@ -129,6 +123,31 @@ export async function getStats() { cached('du_ratings', 10 * 60_000, async () => existsSync(IMDB_RATINGS) ? statSync(IMDB_RATINGS).size : 0, ), + ]); + return { + movie_b: duMovie, + tv_b: duTv, + justwatch_movie_b: duJwMovie, + justwatch_tv_b: duJwTv, + ratings_b: duRatings, + total_b: duMovie + duTv + duJwMovie + duJwTv + duRatings, + }; +} + +// Kicks off disk usage computation in the background to prime the 10 min cache +// before the first admin visitor needs it. Used at server warmup. +export function preloadDiskUsage() { + return getDiskUsage().catch((err) => console.warn('preloadDiskUsage:', err.message)); +} + +export async function getStats() { + // Fast path only: in-memory caches + small file reads + 1-2 wc -l. No du. + // Disk usage is exposed by the separate getDiskUsage() endpoint. + const [movieTotal, tvTotal, cronText, cronLogTail, ratings, mappings] = await Promise.all([ + cached('movie_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'movie.json'))), + cached('tv_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'tv.json'))), + cached('cron_txt', 30_000, async () => (existsSync(CRON_TXT) ? await readFile(CRON_TXT, 'utf8') : '')), + cached('cron_tail', 30_000, () => tailLines(LASTCRON_TXT, 200)), getRatings().catch(() => null), preloadMappings().catch(() => ({ movie: 0, tv: 0 })), ]); @@ -146,14 +165,6 @@ export async function getStats() { movies: { master: movieTotal, with_imdb: mappings.movie }, tv: { master: tvTotal, with_imdb: mappings.tv }, imdb_ratings: ratings ? ratings.size : 0, - disk: { - movie_b: duMovie, - tv_b: duTv, - justwatch_movie_b: duJwMovie, - justwatch_tv_b: duJwTv, - ratings_b: duRatings, - total_b: duMovie + duTv + duJwMovie + duJwTv + duRatings, - }, }, cron: { ...cron, diff --git a/public/admin.js b/public/admin.js index fe335a9..e42f6ed 100644 --- a/public/admin.js +++ b/public/admin.js @@ -72,22 +72,62 @@ function setRefreshStatus(text) { $('#refresh-status').textContent = text; } +function renderDisk(disk) { + const max = Math.max( + disk.movie_b, + disk.tv_b, + disk.justwatch_movie_b, + disk.justwatch_tv_b, + disk.ratings_b, + 1, + ); + const rows = [ + ['Films (détails TMDB)', disk.movie_b, ''], + ['Séries (détails TMDB)', disk.tv_b, 'tv'], + ['Providers films', disk.justwatch_movie_b, 'jwmovie'], + ['Providers séries', disk.justwatch_tv_b, 'jwtv'], + ['IMDb ratings', disk.ratings_b, 'ratings'], + ]; + $('#disk-bars').innerHTML = `${rows + .map( + ([label, b, cls]) => ` +
+ ${label} +
+ ${fmtBytes(b)} +
`, + ) + .join('')} +
+ Total
+ ${fmtBytes(disk.total_b)} +
`; +} + // ---- Dashboard ------------------------------------------------------------ async function loadStats() { setRefreshStatus('Chargement…'); + // Fast endpoint first; disk runs in parallel and fills in once ready. + const statsP = fetch('/admin/api/stats').then((r) => { + if (r.status === 401) { + location.reload(); + throw new Error('auth'); + } + return r.json(); + }); + const diskP = fetch('/admin/api/disk').then((r) => (r.ok ? r.json() : null)); let data; try { - const res = await fetch('/admin/api/stats'); - if (res.status === 401) { - location.reload(); - return; - } - data = await res.json(); + data = await statsP; } catch (err) { - setRefreshStatus(`Erreur : ${err.message}`); + if (err.message !== 'auth') setRefreshStatus(`Erreur : ${err.message}`); return; } + // Schedule disk render independently (don't block the rest of the UI). + diskP.then((disk) => disk && renderDisk(disk)).catch(() => {}); + $('#disk-bars').innerHTML = + '
Calcul des tailles disque…
'; // Cron card const cronEl = $('#cron-status'); @@ -120,39 +160,7 @@ async function loadStats() { $('#process-uptime').textContent = `Uptime : ${fmtUptime(data.process.uptime_s)}`; $('#process-node').textContent = `Node ${data.process.node} · pid ${data.process.pid}`; - // Disk bars - const disk = data.data.disk; - const max = Math.max( - disk.movie_b, - disk.tv_b, - disk.justwatch_movie_b, - disk.justwatch_tv_b, - disk.ratings_b, - 1, - ); - const rows = [ - ['Films (détails TMDB)', disk.movie_b, ''], - ['Séries (détails TMDB)', disk.tv_b, 'tv'], - ['Providers films', disk.justwatch_movie_b, 'jwmovie'], - ['Providers séries', disk.justwatch_tv_b, 'jwtv'], - ['IMDb ratings', disk.ratings_b, 'ratings'], - ]; - $('#disk-bars').innerHTML = - rows - .map( - ([label, b, cls]) => ` -
- ${label} -
- ${fmtBytes(b)} -
- `, - ) - .join('') + - `
- Total
- ${fmtBytes(disk.total_b)} -
`; + // (disk handled in renderDisk via diskP) // Cron summary const sum = data.cron.log_summary; diff --git a/routes/admin.js b/routes/admin.js index 7853fcd..ea04ad6 100644 --- a/routes/admin.js +++ b/routes/admin.js @@ -8,7 +8,7 @@ import { readdir, readFile, stat } from 'node:fs/promises'; import { join } from 'node:path'; import { ADMIN_PASSWORD_HASH, ROOT, TITLE } from '../config.js'; import { verifyPassword } from '../lib/password.js'; -import { getMetricsForUI, getStats } from '../lib/stats.js'; +import { getDiskUsage, getMetricsForUI, getStats } from '../lib/stats.js'; const HIDDEN = new Set([ 'node_modules', @@ -133,6 +133,11 @@ export default async function adminRoutes(fastify) { return getStats(); }); + fastify.get('/admin/api/disk', async (req, reply) => { + if (!requireAuth(req, reply)) return; + return getDiskUsage(); + }); + fastify.get('/admin/api/metrics', async (req, reply) => { if (!requireAuth(req, reply)) return; return getMetricsForUI(); diff --git a/server.js b/server.js index 08ebdae..0aecab7 100644 --- a/server.js +++ b/server.js @@ -10,6 +10,7 @@ import { preloadMappings } from './lib/imdbMapping.js'; import { getRatings } from './lib/imdbRatings.js'; import { httpDuration, httpRequests, imdbRatingsCount, searchWorkers } from './lib/metrics.js'; import { getPool } from './lib/searchEngine.js'; +import { preloadDiskUsage } from './lib/stats.js'; import adminRoutes from './routes/admin.js'; import apiRoutes from './routes/api.js'; import healthRoutes from './routes/health.js'; @@ -84,9 +85,12 @@ fastify.ready().then(async () => { searchWorkers.set({ type: 'tv' }, tv.workers.length); const maps = await preloadMappings(); startWatchers(); + // Disk usage takes ~3 s on cold cache (du -sb on 1.7M files). Run it in + // background so the first admin visitor finds the result already cached. + preloadDiskUsage(); fastify.log.info( { ratings: ratings.size, imdb_movie: maps.movie, imdb_tv: maps.tv }, - 'Warmup complete (data hot-reload watchers started)', + 'Warmup complete (data hot-reload watchers started, disk usage preloading)', ); } catch (err) { fastify.log.warn({ err }, 'Warmup failed');