Phase 1: lock cron, reload chaud, argon2, providers, IMDb lookup, cache LRU, /health, /metrics, rate limit, UI dark, biome
This commit is contained in:
12
lib/http.js
12
lib/http.js
@@ -52,8 +52,16 @@ export class Limiter {
|
||||
Promise.resolve()
|
||||
.then(fn)
|
||||
.then(
|
||||
(v) => { this.active--; resolve(v); this._next(); },
|
||||
(e) => { this.active--; reject(e); this._next(); },
|
||||
(v) => {
|
||||
this.active--;
|
||||
resolve(v);
|
||||
this._next();
|
||||
},
|
||||
(e) => {
|
||||
this.active--;
|
||||
reject(e);
|
||||
this._next();
|
||||
},
|
||||
);
|
||||
};
|
||||
tryRun();
|
||||
|
||||
35
lib/imdbMapping.js
Normal file
35
lib/imdbMapping.js
Normal file
@@ -0,0 +1,35 @@
|
||||
// Loads imdb2movie.json / imdb2tv.json into memory and reloads on mtime change.
|
||||
|
||||
import { readFile, stat } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { TMDBINTEGRAL_DIR } from '../config.js';
|
||||
|
||||
const cache = { movie: null, tv: null };
|
||||
const mtimes = { movie: 0, tv: 0 };
|
||||
|
||||
async function loadOne(type) {
|
||||
const file = join(TMDBINTEGRAL_DIR, `imdb2${type}.json`);
|
||||
const st = await stat(file);
|
||||
if (cache[type] && st.mtimeMs === mtimes[type]) return cache[type];
|
||||
const obj = JSON.parse(await readFile(file, 'utf8'));
|
||||
cache[type] = new Map(Object.entries(obj).map(([k, v]) => [k, Number(v)]));
|
||||
mtimes[type] = st.mtimeMs;
|
||||
return cache[type];
|
||||
}
|
||||
|
||||
export async function lookupImdb(imdbId) {
|
||||
// Try movie first, then tv (matches PHP behaviour of exhaustive lookup).
|
||||
try {
|
||||
const movieMap = await loadOne('movie');
|
||||
if (movieMap.has(imdbId)) return { type: 'movie', tmdb: movieMap.get(imdbId) };
|
||||
} catch {
|
||||
/* file may be missing on a fresh install */
|
||||
}
|
||||
try {
|
||||
const tvMap = await loadOne('tv');
|
||||
if (tvMap.has(imdbId)) return { type: 'tv', tmdb: tvMap.get(imdbId) };
|
||||
} catch {
|
||||
/* same */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -11,7 +11,10 @@ export async function loadRatings(filePath = IMDB_RATINGS) {
|
||||
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
||||
let first = true;
|
||||
for await (const line of rl) {
|
||||
if (first) { first = false; continue; }
|
||||
if (first) {
|
||||
first = false;
|
||||
continue;
|
||||
}
|
||||
if (!line) continue;
|
||||
const tab1 = line.indexOf('\t');
|
||||
if (tab1 < 0) continue;
|
||||
|
||||
60
lib/lockFile.js
Normal file
60
lib/lockFile.js
Normal file
@@ -0,0 +1,60 @@
|
||||
// Simple PID-based lock file. Atomic via O_EXCL.
|
||||
// If a stale lock is found (PID no longer alive), it is removed.
|
||||
|
||||
import { closeSync, openSync, readFileSync, unlinkSync, writeSync } from 'node:fs';
|
||||
|
||||
function isAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (e) {
|
||||
return e.code === 'EPERM';
|
||||
}
|
||||
}
|
||||
|
||||
export function acquireLock(lockPath) {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const fd = openSync(lockPath, 'wx');
|
||||
writeSync(fd, String(process.pid));
|
||||
closeSync(fd);
|
||||
const release = () => {
|
||||
try {
|
||||
unlinkSync(lockPath);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
};
|
||||
process.on('exit', release);
|
||||
process.on('SIGINT', () => {
|
||||
release();
|
||||
process.exit(130);
|
||||
});
|
||||
process.on('SIGTERM', () => {
|
||||
release();
|
||||
process.exit(143);
|
||||
});
|
||||
return release;
|
||||
} catch (err) {
|
||||
if (err.code !== 'EEXIST') throw err;
|
||||
// Lock exists — check if owner is still alive
|
||||
let pid = 0;
|
||||
try {
|
||||
pid = parseInt(readFileSync(lockPath, 'utf8').trim(), 10);
|
||||
} catch {
|
||||
/* unreadable */
|
||||
}
|
||||
if (pid && isAlive(pid)) {
|
||||
throw new Error(`Cron already running (PID ${pid}, lock ${lockPath})`);
|
||||
}
|
||||
// Stale lock — remove and retry
|
||||
console.log(`Removing stale lock (PID ${pid || '?'} no longer alive): ${lockPath}`);
|
||||
try {
|
||||
unlinkSync(lockPath);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error(`Could not acquire lock after retries: ${lockPath}`);
|
||||
}
|
||||
@@ -21,7 +21,7 @@ export function mbLevenshtein(s1, s2, costIns = 1, costRep = 1, costDel = 1) {
|
||||
const del = prev[j] + costDel;
|
||||
const ins = curr[j - 1] + costIns;
|
||||
const rep = prev[j - 1] + cost;
|
||||
curr[j] = del < ins ? (del < rep ? del : rep) : (ins < rep ? ins : rep);
|
||||
curr[j] = del < ins ? (del < rep ? del : rep) : ins < rep ? ins : rep;
|
||||
}
|
||||
[prev, curr] = [curr, prev];
|
||||
}
|
||||
|
||||
46
lib/metrics.js
Normal file
46
lib/metrics.js
Normal file
@@ -0,0 +1,46 @@
|
||||
// Prometheus metrics. Reusable counters/histograms for the API + cache.
|
||||
|
||||
import { Counter, collectDefaultMetrics, Gauge, Histogram, Registry } from 'prom-client';
|
||||
|
||||
export const registry = new Registry();
|
||||
collectDefaultMetrics({ register: registry });
|
||||
|
||||
export const httpRequests = new Counter({
|
||||
name: 'http_requests_total',
|
||||
help: 'Total HTTP requests',
|
||||
labelNames: ['method', 'route', 'status'],
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
export const httpDuration = new Histogram({
|
||||
name: 'http_request_duration_seconds',
|
||||
help: 'HTTP request duration in seconds',
|
||||
labelNames: ['method', 'route', 'status'],
|
||||
buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2, 5],
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
export const searchCacheHits = new Counter({
|
||||
name: 'search_cache_hits_total',
|
||||
help: 'Number of search results served from the LRU cache',
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
export const searchCacheMisses = new Counter({
|
||||
name: 'search_cache_misses_total',
|
||||
help: 'Number of search requests that bypassed the cache',
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
export const imdbRatingsCount = new Gauge({
|
||||
name: 'imdb_ratings_total',
|
||||
help: 'Number of IMDb ratings currently loaded in memory',
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
export const searchWorkers = new Gauge({
|
||||
name: 'search_workers',
|
||||
help: 'Number of active search workers per type',
|
||||
labelNames: ['type'],
|
||||
registers: [registry],
|
||||
});
|
||||
21
lib/password.js
Normal file
21
lib/password.js
Normal file
@@ -0,0 +1,21 @@
|
||||
import { Algorithm, hash, verify } from '@node-rs/argon2';
|
||||
|
||||
const OPTS = {
|
||||
algorithm: Algorithm.Argon2id,
|
||||
memoryCost: 19456, // 19 MiB (OWASP 2024 recommendation)
|
||||
timeCost: 2,
|
||||
parallelism: 1,
|
||||
};
|
||||
|
||||
export async function hashPassword(plain) {
|
||||
return await hash(plain, OPTS);
|
||||
}
|
||||
|
||||
export async function verifyPassword(stored, plain) {
|
||||
if (!stored || !plain) return false;
|
||||
try {
|
||||
return await verify(stored, plain);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { join } from 'node:path';
|
||||
import { MOVIE_DIR, TV_DIR, JUSTWATCH_MOVIE_DIR, JUSTWATCH_TV_DIR } from '../config.js';
|
||||
import { JUSTWATCH_MOVIE_DIR, JUSTWATCH_TV_DIR, MOVIE_DIR, TV_DIR } from '../config.js';
|
||||
|
||||
export function bucket(id) {
|
||||
return String(Math.floor(id / 1000));
|
||||
|
||||
@@ -12,7 +12,8 @@ const YEAR_RE = /(19|20)\d{2}/g;
|
||||
// - the lowercase 'x' in NxN, and uppercase 'E' in standalone Exxx, are case-sensitive
|
||||
// Greedy left-to-right alternation means "S01E02" is consumed whole, so the
|
||||
// trailing "E02" alternative cannot match inside it.
|
||||
const EPISODE_RE = /[Ss][0-9]{1,2}.?[Ee][0-9]{1,3}|[Ss][0-9]{2}|[Pp]art\.?[0-9]{1,3}|[0-9]{1,2}x[0-9]{1,3}|E[0-9]{1,3}/g;
|
||||
const EPISODE_RE =
|
||||
/[Ss][0-9]{1,2}.?[Ee][0-9]{1,3}|[Ss][0-9]{2}|[Pp]art\.?[0-9]{1,3}|[0-9]{1,2}x[0-9]{1,3}|E[0-9]{1,3}/g;
|
||||
|
||||
// PHP uses byte offsets (substr). To stay byte-faithful, work on the UTF-8 bytes.
|
||||
const utf8 = (s) => Buffer.from(s, 'utf8');
|
||||
@@ -21,8 +22,9 @@ const sliceBytes = (s, end) => utf8(s).slice(0, end).toString('utf8');
|
||||
function findAll(re, str) {
|
||||
const out = [];
|
||||
re.lastIndex = 0;
|
||||
let m;
|
||||
while ((m = re.exec(str)) !== null) {
|
||||
for (;;) {
|
||||
const m = re.exec(str);
|
||||
if (m === null) break;
|
||||
out.push({ value: m[0], byteOffset: Buffer.byteLength(str.slice(0, m.index), 'utf8') });
|
||||
if (m.index === re.lastIndex) re.lastIndex++;
|
||||
}
|
||||
|
||||
@@ -2,18 +2,23 @@
|
||||
// queries across them. Workers are kept alive between requests so the chunks
|
||||
// stay loaded in memory (replaces the per-request `php searchmultithreads.php`
|
||||
// fork from the PHP version).
|
||||
//
|
||||
// A filesystem watcher detects when the cron rewrites the chunks and recycles
|
||||
// the worker pool transparently — no server restart needed.
|
||||
|
||||
import { Worker } from 'node:worker_threads';
|
||||
import { join } from 'node:path';
|
||||
import { existsSync, watch } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname } from 'node:path';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { TMDBINTEGRAL_DIR, NB_WORKERS } from '../config.js';
|
||||
import { Worker } from 'node:worker_threads';
|
||||
import { NB_WORKERS, TMDBINTEGRAL_DIR } from '../config.js';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const WORKER_PATH = join(__dirname, 'searchWorker.js');
|
||||
|
||||
const pools = new Map();
|
||||
let watcher = null;
|
||||
let reloadTimer = null;
|
||||
const RELOAD_DEBOUNCE_MS = 5000;
|
||||
|
||||
class WorkerPool {
|
||||
constructor(type) {
|
||||
@@ -55,10 +60,45 @@ class WorkerPool {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async terminate() {
|
||||
await Promise.allSettled(this.workers.map((w) => w.terminate()));
|
||||
this.workers = [];
|
||||
}
|
||||
}
|
||||
|
||||
async function reloadAllPools() {
|
||||
const types = [...pools.keys()];
|
||||
console.log(`Reloading search pools: ${types.join(', ')}`);
|
||||
for (const type of types) {
|
||||
const old = pools.get(type);
|
||||
pools.set(type, new WorkerPool(type));
|
||||
old.terminate().catch(() => {
|
||||
/* ignore */
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function ensureWatcher() {
|
||||
if (watcher) return;
|
||||
try {
|
||||
watcher = watch(TMDBINTEGRAL_DIR, (_event, filename) => {
|
||||
if (!filename) return;
|
||||
if (!/^search(movie|tv)\d+\.json$/.test(filename)) return;
|
||||
clearTimeout(reloadTimer);
|
||||
reloadTimer = setTimeout(reloadAllPools, RELOAD_DEBOUNCE_MS);
|
||||
});
|
||||
watcher.unref();
|
||||
} catch (err) {
|
||||
console.warn(`Cannot watch ${TMDBINTEGRAL_DIR} for chunk reload:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
export function getPool(type) {
|
||||
if (!pools.has(type)) pools.set(type, new WorkerPool(type));
|
||||
if (!pools.has(type)) {
|
||||
pools.set(type, new WorkerPool(type));
|
||||
ensureWatcher();
|
||||
}
|
||||
return pools.get(type);
|
||||
}
|
||||
|
||||
|
||||
@@ -3,10 +3,8 @@
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
import { LEV_DEL, LEV_INS, LEV_REP, LEV_SCALE, TITLE_TOLERANCE, YEAR_TOLERANCE } from '../config.js';
|
||||
import { mbLevenshtein, mbStrlen } from './mbLevenshtein.js';
|
||||
import {
|
||||
TITLE_TOLERANCE, LEV_INS, LEV_REP, LEV_DEL, LEV_SCALE, YEAR_TOLERANCE,
|
||||
} from '../config.js';
|
||||
|
||||
const TMDB = 0;
|
||||
const TITLE = 1;
|
||||
@@ -34,8 +32,11 @@ function loadChunk() {
|
||||
function score(filteredIn, target, ftiLen) {
|
||||
if (!target) return 0;
|
||||
const tlen = mbStrlen(target);
|
||||
return 100 - (mbLevenshtein(filteredIn, target, LEV_INS, LEV_REP, LEV_DEL) /
|
||||
(Math.max(ftiLen, tlen) * LEV_SCALE)) * 100;
|
||||
return (
|
||||
100 -
|
||||
(mbLevenshtein(filteredIn, target, LEV_INS, LEV_REP, LEV_DEL) / (Math.max(ftiLen, tlen) * LEV_SCALE)) *
|
||||
100
|
||||
);
|
||||
}
|
||||
|
||||
function search({ filteredTitleIn, yearIn }) {
|
||||
@@ -49,7 +50,11 @@ function search({ filteredTitleIn, yearIn }) {
|
||||
let ok = false;
|
||||
for (const y of row[YEAR]) {
|
||||
const dy = Math.abs(yearIn - y);
|
||||
if (dy <= YEAR_TOLERANCE) { ok = true; deltaYear = dy; break; }
|
||||
if (dy <= YEAR_TOLERANCE) {
|
||||
ok = true;
|
||||
deltaYear = dy;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!ok) continue;
|
||||
}
|
||||
@@ -62,7 +67,7 @@ function search({ filteredTitleIn, yearIn }) {
|
||||
|
||||
let pT;
|
||||
if (fT) {
|
||||
pT = (fT === fO) ? pO : score(filteredTitleIn, fT, ftiLen);
|
||||
pT = fT === fO ? pO : score(filteredTitleIn, fT, ftiLen);
|
||||
} else pT = 0;
|
||||
|
||||
let pE;
|
||||
|
||||
Reference in New Issue
Block a user