Dashboard admin : split stats/disk + preload disk au warmup + wc -l (1.85s -> 40ms)

2026-04-24 08:48:29 +02:00
parent bd8e4e5228
commit 54b6cc453e
4 changed files with 100 additions and 72 deletions
--- a/lib/stats.js
+++ b/lib/stats.js
@@ -45,11 +45,18 @@ async function cached(key, ttlMs, fn) {

 async function countLines(file) {
  if (!existsSync(file)) return 0;
+  // wc -l is ~10x faster than Node readline on large files. Falls back to
+  // readline if wc is unavailable (non-Linux dev env).
+  try {
+    const { stdout } = await execFile('wc', ['-l', file], { maxBuffer: 1024 * 1024 });
+    return parseInt(stdout.trim().split(/\s+/)[0], 10) || 0;
+  } catch {
    let n = 0;
    const rl = createInterface({ input: createReadStream(file), crlfDelay: Infinity });
    for await (const _ of rl) n++;
    return n;
  }
+}

 async function tailLines(file, n) {
  if (!existsSync(file)) return [];
@@ -104,24 +111,11 @@ async function getCounterValue(name) {
  return arr.values.reduce((sum, v) => sum + (v.value || 0), 0);
 }

-export async function getStats() {
-  const [
-    movieTotal,
-    tvTotal,
-    cronText,
-    cronLogTail,
-    duMovie,
-    duTv,
-    duJwMovie,
-    duJwTv,
-    duRatings,
-    ratings,
-    mappings,
-  ] = await Promise.all([
-    cached('movie_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'movie.json'))),
-    cached('tv_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'tv.json'))),
-    cached('cron_txt', 30_000, async () => (existsSync(CRON_TXT) ? await readFile(CRON_TXT, 'utf8') : '')),
-    cached('cron_tail', 30_000, () => tailLines(LASTCRON_TXT, 200)),
+// Disk usage is the only slow part (du -sb on 1.7M files = ~2-3s cold,
+// then 10 min cache). Exposed separately so the dashboard can show fast
+// stats first and fill in disk asynchronously.
+export async function getDiskUsage() {
+  const [duMovie, duTv, duJwMovie, duJwTv, duRatings] = await Promise.all([
    cached('du_movie', 10 * 60_000, () => diskUsage(MOVIE_DIR)),
    cached('du_tv', 10 * 60_000, () => diskUsage(TV_DIR)),
    cached('du_jwmovie', 10 * 60_000, () => diskUsage(JUSTWATCH_MOVIE_DIR)),
@@ -129,6 +123,31 @@ export async function getStats() {
    cached('du_ratings', 10 * 60_000, async () =>
      existsSync(IMDB_RATINGS) ? statSync(IMDB_RATINGS).size : 0,
    ),
+  ]);
+  return {
+    movie_b: duMovie,
+    tv_b: duTv,
+    justwatch_movie_b: duJwMovie,
+    justwatch_tv_b: duJwTv,
+    ratings_b: duRatings,
+    total_b: duMovie + duTv + duJwMovie + duJwTv + duRatings,
+  };
+}
+
+// Kicks off disk usage computation in the background to prime the 10 min cache
+// before the first admin visitor needs it. Used at server warmup.
+export function preloadDiskUsage() {
+  return getDiskUsage().catch((err) => console.warn('preloadDiskUsage:', err.message));
+}
+
+export async function getStats() {
+  // Fast path only: in-memory caches + small file reads + 1-2 wc -l. No du.
+  // Disk usage is exposed by the separate getDiskUsage() endpoint.
+  const [movieTotal, tvTotal, cronText, cronLogTail, ratings, mappings] = await Promise.all([
+    cached('movie_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'movie.json'))),
+    cached('tv_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'tv.json'))),
+    cached('cron_txt', 30_000, async () => (existsSync(CRON_TXT) ? await readFile(CRON_TXT, 'utf8') : '')),
+    cached('cron_tail', 30_000, () => tailLines(LASTCRON_TXT, 200)),
    getRatings().catch(() => null),
    preloadMappings().catch(() => ({ movie: 0, tv: 0 })),
  ]);
@@ -146,14 +165,6 @@ export async function getStats() {
      movies: { master: movieTotal, with_imdb: mappings.movie },
      tv: { master: tvTotal, with_imdb: mappings.tv },
      imdb_ratings: ratings ? ratings.size : 0,
-      disk: {
-        movie_b: duMovie,
-        tv_b: duTv,
-        justwatch_movie_b: duJwMovie,
-        justwatch_tv_b: duJwTv,
-        ratings_b: duRatings,
-        total_b: duMovie + duTv + duJwMovie + duJwTv + duRatings,
-      },
    },
    cron: {
      ...cron,
--- a/public/admin.js
+++ b/public/admin.js
@@ -72,22 +72,62 @@ function setRefreshStatus(text) {
  $('#refresh-status').textContent = text;
 }

+function renderDisk(disk) {
+  const max = Math.max(
+    disk.movie_b,
+    disk.tv_b,
+    disk.justwatch_movie_b,
+    disk.justwatch_tv_b,
+    disk.ratings_b,
+    1,
+  );
+  const rows = [
+    ['Films (détails TMDB)', disk.movie_b, ''],
+    ['Séries (détails TMDB)', disk.tv_b, 'tv'],
+    ['Providers films', disk.justwatch_movie_b, 'jwmovie'],
+    ['Providers séries', disk.justwatch_tv_b, 'jwtv'],
+    ['IMDb ratings', disk.ratings_b, 'ratings'],
+  ];
+  $('#disk-bars').innerHTML = `${rows
+    .map(
+      ([label, b, cls]) => `
+      <div class="disk-row">
+        <span class="label">${label}</span>
+        <div class="disk-bar ${cls}"><span style="width:${(b / max) * 100}%"></span></div>
+        <span class="size">${fmtBytes(b)}</span>
+      </div>`,
+    )
+    .join('')}
+    <div class="disk-row" style="margin-top:6px;border-top:1px solid var(--border);padding-top:10px">
+      <span class="label" style="color:var(--text)">Total</span><div></div>
+      <span class="size" style="color:var(--text);font-weight:600">${fmtBytes(disk.total_b)}</span>
+    </div>`;
+}
+
 // ---- Dashboard ------------------------------------------------------------

 async function loadStats() {
  setRefreshStatus('Chargement…');
+  // Fast endpoint first; disk runs in parallel and fills in once ready.
+  const statsP = fetch('/admin/api/stats').then((r) => {
+    if (r.status === 401) {
+      location.reload();
+      throw new Error('auth');
+    }
+    return r.json();
+  });
+  const diskP = fetch('/admin/api/disk').then((r) => (r.ok ? r.json() : null));
  let data;
  try {
-    const res = await fetch('/admin/api/stats');
-    if (res.status === 401) {
-      location.reload();
-      return;
-    }
-    data = await res.json();
+    data = await statsP;
  } catch (err) {
-    setRefreshStatus(`Erreur : ${err.message}`);
+    if (err.message !== 'auth') setRefreshStatus(`Erreur : ${err.message}`);
    return;
  }
+  // Schedule disk render independently (don't block the rest of the UI).
+  diskP.then((disk) => disk && renderDisk(disk)).catch(() => {});
+  $('#disk-bars').innerHTML =
+    '<div class="status loading" style="padding:20px">Calcul des tailles disque…</div>';

  // Cron card
  const cronEl = $('#cron-status');
@@ -120,39 +160,7 @@ async function loadStats() {
  $('#process-uptime').textContent = `Uptime : ${fmtUptime(data.process.uptime_s)}`;
  $('#process-node').textContent = `Node ${data.process.node} · pid ${data.process.pid}`;

-  // Disk bars
-  const disk = data.data.disk;
-  const max = Math.max(
-    disk.movie_b,
-    disk.tv_b,
-    disk.justwatch_movie_b,
-    disk.justwatch_tv_b,
-    disk.ratings_b,
-    1,
-  );
-  const rows = [
-    ['Films (détails TMDB)', disk.movie_b, ''],
-    ['Séries (détails TMDB)', disk.tv_b, 'tv'],
-    ['Providers films', disk.justwatch_movie_b, 'jwmovie'],
-    ['Providers séries', disk.justwatch_tv_b, 'jwtv'],
-    ['IMDb ratings', disk.ratings_b, 'ratings'],
-  ];
-  $('#disk-bars').innerHTML =
-    rows
-      .map(
-        ([label, b, cls]) => `
-    <div class="disk-row">
-      <span class="label">${label}</span>
-      <div class="disk-bar ${cls}"><span style="width:${(b / max) * 100}%"></span></div>
-      <span class="size">${fmtBytes(b)}</span>
-    </div>
-  `,
-      )
-      .join('') +
-    `<div class="disk-row" style="margin-top:6px;border-top:1px solid var(--border);padding-top:10px">
-    <span class="label" style="color:var(--text)">Total</span><div></div>
-    <span class="size" style="color:var(--text);font-weight:600">${fmtBytes(disk.total_b)}</span>
-  </div>`;
+  // (disk handled in renderDisk via diskP)

  // Cron summary
  const sum = data.cron.log_summary;
--- a/routes/admin.js
+++ b/routes/admin.js
@@ -8,7 +8,7 @@ import { readdir, readFile, stat } from 'node:fs/promises';
 import { join } from 'node:path';
 import { ADMIN_PASSWORD_HASH, ROOT, TITLE } from '../config.js';
 import { verifyPassword } from '../lib/password.js';
-import { getMetricsForUI, getStats } from '../lib/stats.js';
+import { getDiskUsage, getMetricsForUI, getStats } from '../lib/stats.js';

 const HIDDEN = new Set([
  'node_modules',
@@ -133,6 +133,11 @@ export default async function adminRoutes(fastify) {
    return getStats();
  });

+  fastify.get('/admin/api/disk', async (req, reply) => {
+    if (!requireAuth(req, reply)) return;
+    return getDiskUsage();
+  });
+
  fastify.get('/admin/api/metrics', async (req, reply) => {
    if (!requireAuth(req, reply)) return;
    return getMetricsForUI();
--- a/server.js
+++ b/server.js
@@ -10,6 +10,7 @@ import { preloadMappings } from './lib/imdbMapping.js';
 import { getRatings } from './lib/imdbRatings.js';
 import { httpDuration, httpRequests, imdbRatingsCount, searchWorkers } from './lib/metrics.js';
 import { getPool } from './lib/searchEngine.js';
+import { preloadDiskUsage } from './lib/stats.js';
 import adminRoutes from './routes/admin.js';
 import apiRoutes from './routes/api.js';
 import healthRoutes from './routes/health.js';
@@ -84,9 +85,12 @@ fastify.ready().then(async () => {
    searchWorkers.set({ type: 'tv' }, tv.workers.length);
    const maps = await preloadMappings();
    startWatchers();
+    // Disk usage takes ~3 s on cold cache (du -sb on 1.7M files). Run it in
+    // background so the first admin visitor finds the result already cached.
+    preloadDiskUsage();
    fastify.log.info(
      { ratings: ratings.size, imdb_movie: maps.movie, imdb_tv: maps.tv },
-      'Warmup complete (data hot-reload watchers started)',
+      'Warmup complete (data hot-reload watchers started, disk usage preloading)',
    );
  } catch (err) {
    fastify.log.warn({ err }, 'Warmup failed');