Portage complet PHP/Bash vers Node.js (Fastify + worker_threads)

cron/justwatchSync.js

@@ -0,0 +1,93 @@
+// Port of tmdbintegral/justwatch.php
+
+import { createReadStream, existsSync, readdirSync, unlinkSync } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { createInterface } from 'node:readline';
+import { join } from 'node:path';
+import {
+  TMDBINTEGRAL_DIR, JUSTWATCH_MOVIE_DIR, JUSTWATCH_TV_DIR, TMDB_API_KEY, TMDB_API_BASE,
+} from '../config.js';
+import { Limiter } from '../lib/http.js';
+import { justwatchDir, justwatchPath, bucket } from '../lib/paths.js';
+
+const DOWNLOAD_CONCURRENCY = 16;
+
+async function readMasterIds(type) {
+  const file = join(TMDBINTEGRAL_DIR, `${type}.json`);
+  const ids = [];
+  const stream = createReadStream(file, { encoding: 'utf8' });
+  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+  for await (const line of rl) {
+    if (!line) continue;
+    try {
+      const obj = JSON.parse(line);
+      if (typeof obj.id === 'number') ids.push(obj.id);
+    } catch { /* ignore */ }
+  }
+  return ids;
+}
+
+async function ensureDir(dir) {
+  if (!existsSync(dir)) await mkdir(dir, { recursive: true });
+}
+
+async function downloadProvider(type, id) {
+  const dir = justwatchDir(type, id);
+  await ensureDir(dir);
+  const path = justwatchPath(type, id);
+  const url = `${TMDB_API_BASE}/${type}/${id}/watch/providers?api_key=${TMDB_API_KEY}`;
+  console.log(`Downloading: "justwatch${type}/${bucket(id)}/${id}.json"`);
+  const res = await fetch(url);
+  if (!res.ok) {
+    console.log(`Failed to retrieve TMDb data: "${url}"`);
+    return;
+  }
+  const text = await res.text();
+  await writeFile(path, text);
+}
+
+function removeOrphans(type, ids) {
+  const baseDir = type === 'movie' ? JUSTWATCH_MOVIE_DIR : JUSTWATCH_TV_DIR;
+  const expected = new Set(ids);
+  let buckets;
+  try { buckets = readdirSync(baseDir); } catch { return; }
+  for (const b of buckets) {
+    let entries;
+    try { entries = readdirSync(join(baseDir, b)); } catch { continue; }
+    for (const fname of entries) {
+      if (!fname.endsWith('.json')) continue;
+      const id = parseInt(fname.slice(0, -5), 10);
+      if (!Number.isInteger(id)) continue;
+      if (!expected.has(id)) {
+        const p = join(baseDir, b, fname);
+        console.log(`Removing: "justwatch${type}/${b}/${fname}"`);
+        try { unlinkSync(p); } catch { /* ignore */ }
+      }
+    }
+  }
+}
+
+export async function syncType(type) {
+  const ids = await readMasterIds(type);
+  const limiter = new Limiter(DOWNLOAD_CONCURRENCY);
+  const tasks = [];
+  for (const id of ids) {
+    if (existsSync(justwatchPath(type, id))) continue;
+    tasks.push(limiter.run(() => downloadProvider(type, id)));
+  }
+  await Promise.allSettled(tasks);
+  ids.sort((a, b) => a - b);
+  removeOrphans(type, ids);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const type = process.argv[2];
+  if (type !== 'movie' && type !== 'tv') {
+    console.error('Usage: node cron/justwatchSync.js movie|tv');
+    process.exit(1);
+  }
+  syncType(type).catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });
+}