96 lines
3.1 KiB
JavaScript
96 lines
3.1 KiB
JavaScript
// Port of tmdbintegral/justwatch.php
|
|
|
|
import { createReadStream, existsSync, readdirSync, unlinkSync } from 'node:fs';
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import { createInterface } from 'node:readline';
|
|
import { join } from 'node:path';
|
|
import {
|
|
TMDBINTEGRAL_DIR, JUSTWATCH_MOVIE_DIR, JUSTWATCH_TV_DIR, TMDB_API_KEY, TMDB_API_BASE,
|
|
} from '../config.js';
|
|
import { Limiter } from '../lib/http.js';
|
|
import { justwatchDir, justwatchPath, bucket } from '../lib/paths.js';
|
|
|
|
const DOWNLOAD_CONCURRENCY = 16;
|
|
|
|
async function readMasterIds(type) {
|
|
const file = join(TMDBINTEGRAL_DIR, `${type}.json`);
|
|
const ids = [];
|
|
const stream = createReadStream(file, { encoding: 'utf8' });
|
|
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
|
for await (const line of rl) {
|
|
if (!line) continue;
|
|
try {
|
|
const obj = JSON.parse(line);
|
|
if (typeof obj.id === 'number') ids.push(obj.id);
|
|
} catch { /* ignore */ }
|
|
}
|
|
return ids;
|
|
}
|
|
|
|
async function ensureDir(dir) {
|
|
if (!existsSync(dir)) await mkdir(dir, { recursive: true });
|
|
}
|
|
|
|
async function downloadProvider(type, id) {
|
|
const dir = justwatchDir(type, id);
|
|
await ensureDir(dir);
|
|
const path = justwatchPath(type, id);
|
|
const url = `${TMDB_API_BASE}/${type}/${id}/watch/providers?api_key=${TMDB_API_KEY}`;
|
|
console.log(`Downloading: "justwatch${type}/${bucket(id)}/${id}.json"`);
|
|
const res = await fetch(url);
|
|
if (!res.ok) {
|
|
console.log(`Failed to retrieve TMDb data: "${url}"`);
|
|
return;
|
|
}
|
|
const text = await res.text();
|
|
await writeFile(path, text);
|
|
}
|
|
|
|
function removeOrphans(type, ids) {
|
|
const baseDir = type === 'movie' ? JUSTWATCH_MOVIE_DIR : JUSTWATCH_TV_DIR;
|
|
const expected = new Set(ids);
|
|
let buckets;
|
|
try { buckets = readdirSync(baseDir); } catch { return; }
|
|
for (const b of buckets) {
|
|
let entries;
|
|
try { entries = readdirSync(join(baseDir, b)); } catch { continue; }
|
|
for (const fname of entries) {
|
|
if (!fname.endsWith('.json')) continue;
|
|
const id = parseInt(fname.slice(0, -5), 10);
|
|
if (!Number.isInteger(id)) continue;
|
|
if (!expected.has(id)) {
|
|
const p = join(baseDir, b, fname);
|
|
console.log(`Removing: "justwatch${type}/${b}/${fname}"`);
|
|
try { unlinkSync(p); } catch { /* ignore */ }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function syncType(type) {
|
|
const baseDir = type === 'movie' ? JUSTWATCH_MOVIE_DIR : JUSTWATCH_TV_DIR;
|
|
await mkdir(baseDir, { recursive: true });
|
|
const ids = await readMasterIds(type);
|
|
const limiter = new Limiter(DOWNLOAD_CONCURRENCY);
|
|
const tasks = [];
|
|
for (const id of ids) {
|
|
if (existsSync(justwatchPath(type, id))) continue;
|
|
tasks.push(limiter.run(() => downloadProvider(type, id)));
|
|
}
|
|
await Promise.allSettled(tasks);
|
|
ids.sort((a, b) => a - b);
|
|
removeOrphans(type, ids);
|
|
}
|
|
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
const type = process.argv[2];
|
|
if (type !== 'movie' && type !== 'tv') {
|
|
console.error('Usage: node cron/justwatchSync.js movie|tv');
|
|
process.exit(1);
|
|
}
|
|
syncType(type).catch((err) => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|
|
}
|