// Port of tmdbintegral/justwatch.php import { createReadStream, existsSync, readdirSync, unlinkSync } from 'node:fs'; import { mkdir, writeFile } from 'node:fs/promises'; import { createInterface } from 'node:readline'; import { join } from 'node:path'; import { TMDBINTEGRAL_DIR, JUSTWATCH_MOVIE_DIR, JUSTWATCH_TV_DIR, TMDB_API_KEY, TMDB_API_BASE, } from '../config.js'; import { Limiter } from '../lib/http.js'; import { justwatchDir, justwatchPath, bucket } from '../lib/paths.js'; const DOWNLOAD_CONCURRENCY = 16; async function readMasterIds(type) { const file = join(TMDBINTEGRAL_DIR, `${type}.json`); const ids = []; const stream = createReadStream(file, { encoding: 'utf8' }); const rl = createInterface({ input: stream, crlfDelay: Infinity }); for await (const line of rl) { if (!line) continue; try { const obj = JSON.parse(line); if (typeof obj.id === 'number') ids.push(obj.id); } catch { /* ignore */ } } return ids; } async function ensureDir(dir) { if (!existsSync(dir)) await mkdir(dir, { recursive: true }); } async function downloadProvider(type, id) { const dir = justwatchDir(type, id); await ensureDir(dir); const path = justwatchPath(type, id); const url = `${TMDB_API_BASE}/${type}/${id}/watch/providers?api_key=${TMDB_API_KEY}`; console.log(`Downloading: "justwatch${type}/${bucket(id)}/${id}.json"`); const res = await fetch(url); if (!res.ok) { console.log(`Failed to retrieve TMDb data: "${url}"`); return; } const text = await res.text(); await writeFile(path, text); } function removeOrphans(type, ids) { const baseDir = type === 'movie' ? JUSTWATCH_MOVIE_DIR : JUSTWATCH_TV_DIR; const expected = new Set(ids); let buckets; try { buckets = readdirSync(baseDir); } catch { return; } for (const b of buckets) { let entries; try { entries = readdirSync(join(baseDir, b)); } catch { continue; } for (const fname of entries) { if (!fname.endsWith('.json')) continue; const id = parseInt(fname.slice(0, -5), 10); if (!Number.isInteger(id)) continue; if (!expected.has(id)) { const p = join(baseDir, b, fname); console.log(`Removing: "justwatch${type}/${b}/${fname}"`); try { unlinkSync(p); } catch { /* ignore */ } } } } } export async function syncType(type) { const baseDir = type === 'movie' ? JUSTWATCH_MOVIE_DIR : JUSTWATCH_TV_DIR; await mkdir(baseDir, { recursive: true }); const ids = await readMasterIds(type); const limiter = new Limiter(DOWNLOAD_CONCURRENCY); const tasks = []; for (const id of ids) { if (existsSync(justwatchPath(type, id))) continue; tasks.push(limiter.run(() => downloadProvider(type, id))); } await Promise.allSettled(tasks); ids.sort((a, b) => a - b); removeOrphans(type, ids); } if (import.meta.url === `file://${process.argv[1]}`) { const type = process.argv[2]; if (type !== 'movie' && type !== 'tv') { console.error('Usage: node cron/justwatchSync.js movie|tv'); process.exit(1); } syncType(type).catch((err) => { console.error(err); process.exit(1); }); }