UNFR/proxy_tmdb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dashboard admin : split stats/disk + preload disk au warmup + wc -l (1.85s -> 40ms)

Browse Source
This commit is contained in:
unfr
2026-04-24 08:48:29 +02:00
parent bd8e4e5228
commit 54b6cc453e
4 changed files with 100 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
lib/stats.js
View File

@@ -45,10 +45,17 @@ async function cached(key, ttlMs, fn) {
async function countLines(file) {
if (!existsSync(file)) return 0;
let n = 0;
const rl = createInterface({ input: createReadStream(file), crlfDelay: Infinity });
for await (const _ of rl) n++;
return n;
// wc -l is ~10x faster than Node readline on large files. Falls back to
// readline if wc is unavailable (non-Linux dev env).
try {
const { stdout } = await execFile('wc', ['-l', file], { maxBuffer: 1024 * 1024 });
return parseInt(stdout.trim().split(/\s+/)[0], 10) || 0;
} catch {
let n = 0;
const rl = createInterface({ input: createReadStream(file), crlfDelay: Infinity });
for await (const _ of rl) n++;
return n;
}
}
async function tailLines(file, n) {
@@ -104,24 +111,11 @@ async function getCounterValue(name) {
return arr.values.reduce((sum, v) => sum + (v.value || 0), 0);
}
export async function getStats() {
const [
movieTotal,
tvTotal,
cronText,
cronLogTail,
duMovie,
duTv,
duJwMovie,
duJwTv,
duRatings,
ratings,
mappings,
] = await Promise.all([
cached('movie_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'movie.json'))),
cached('tv_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'tv.json'))),
cached('cron_txt', 30_000, async () => (existsSync(CRON_TXT) ? await readFile(CRON_TXT, 'utf8') : '')),
cached('cron_tail', 30_000, () => tailLines(LASTCRON_TXT, 200)),
// Disk usage is the only slow part (du -sb on 1.7M files = ~2-3s cold,
// then 10 min cache). Exposed separately so the dashboard can show fast
// stats first and fill in disk asynchronously.
export async function getDiskUsage() {
const [duMovie, duTv, duJwMovie, duJwTv, duRatings] = await Promise.all([
cached('du_movie', 10 * 60_000, () => diskUsage(MOVIE_DIR)),
cached('du_tv', 10 * 60_000, () => diskUsage(TV_DIR)),
cached('du_jwmovie', 10 * 60_000, () => diskUsage(JUSTWATCH_MOVIE_DIR)),
@@ -129,6 +123,31 @@ export async function getStats() {
cached('du_ratings', 10 * 60_000, async () =>
existsSync(IMDB_RATINGS) ? statSync(IMDB_RATINGS).size : 0,
),
]);
return {
movie_b: duMovie,
tv_b: duTv,
justwatch_movie_b: duJwMovie,
justwatch_tv_b: duJwTv,
ratings_b: duRatings,
total_b: duMovie + duTv + duJwMovie + duJwTv + duRatings,
};
}
// Kicks off disk usage computation in the background to prime the 10 min cache
// before the first admin visitor needs it. Used at server warmup.
export function preloadDiskUsage() {
return getDiskUsage().catch((err) => console.warn('preloadDiskUsage:', err.message));
}
export async function getStats() {
// Fast path only: in-memory caches + small file reads + 1-2 wc -l. No du.
// Disk usage is exposed by the separate getDiskUsage() endpoint.
const [movieTotal, tvTotal, cronText, cronLogTail, ratings, mappings] = await Promise.all([
cached('movie_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'movie.json'))),
cached('tv_total', 5 * 60_000, () => countLines(join(TMDBINTEGRAL_DIR, 'tv.json'))),
cached('cron_txt', 30_000, async () => (existsSync(CRON_TXT) ? await readFile(CRON_TXT, 'utf8') : '')),
cached('cron_tail', 30_000, () => tailLines(LASTCRON_TXT, 200)),
getRatings().catch(() => null),
preloadMappings().catch(() => ({ movie: 0, tv: 0 })),
]);
@@ -146,14 +165,6 @@ export async function getStats() {
movies: { master: movieTotal, with_imdb: mappings.movie },
tv: { master: tvTotal, with_imdb: mappings.tv },
imdb_ratings: ratings ? ratings.size : 0,
disk: {
movie_b: duMovie,
tv_b: duTv,
justwatch_movie_b: duJwMovie,
justwatch_tv_b: duJwTv,
ratings_b: duRatings,
total_b: duMovie + duTv + duJwMovie + duJwTv + duRatings,
},
},
cron: {
...cron,