First dev release of v2.0.0
This commit is contained in:
360
src/metadata/anilist.js
Normal file
360
src/metadata/anilist.js
Normal file
@@ -0,0 +1,360 @@
|
||||
const sqlite3 = require('sqlite3').verbose();
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
// --- CONFIGURATION ---
|
||||
const DB_PATH = path.join(__dirname, 'anilist_anime.db');
|
||||
const REQUESTS_PER_MINUTE = 20; // 20 RPM is safe (AniList limit is 90)
|
||||
const DELAY_MS = (60000 / REQUESTS_PER_MINUTE);
|
||||
const FEATURED_REFRESH_RATE = 8 * 60 * 1000; // 8 Minutes
|
||||
|
||||
// Ensure directory exists
|
||||
const dir = path.dirname(DB_PATH);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
const db = new sqlite3.Database(DB_PATH);
|
||||
|
||||
// --- DATABASE SETUP ---
|
||||
function initDB() {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.serialize(() => {
|
||||
// 1. Anime Tables
|
||||
db.run(`CREATE TABLE IF NOT EXISTS anime (id INTEGER PRIMARY KEY, title TEXT, updatedAt INTEGER, full_data JSON)`);
|
||||
db.run(`CREATE TABLE IF NOT EXISTS trending (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
|
||||
db.run(`CREATE TABLE IF NOT EXISTS top_airing (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
|
||||
|
||||
// 2. Books Tables (Manga/LN)
|
||||
db.run(`CREATE TABLE IF NOT EXISTS books (id INTEGER PRIMARY KEY, title TEXT, updatedAt INTEGER, full_data JSON)`);
|
||||
db.run(`CREATE TABLE IF NOT EXISTS trending_books (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
|
||||
db.run(`CREATE TABLE IF NOT EXISTS popular_books (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// --- QUERIES ---
|
||||
|
||||
// Exhaustive list of fields
|
||||
const MEDIA_FIELDS = `
|
||||
id
|
||||
idMal
|
||||
title { romaji english native userPreferred }
|
||||
type
|
||||
format
|
||||
status
|
||||
description
|
||||
startDate { year month day }
|
||||
endDate { year month day }
|
||||
season
|
||||
seasonYear
|
||||
seasonInt
|
||||
episodes
|
||||
duration
|
||||
chapters
|
||||
volumes
|
||||
countryOfOrigin
|
||||
isLicensed
|
||||
source
|
||||
hashtag
|
||||
trailer { id site thumbnail }
|
||||
updatedAt
|
||||
coverImage { extraLarge large medium color }
|
||||
bannerImage
|
||||
genres
|
||||
synonyms
|
||||
averageScore
|
||||
meanScore
|
||||
popularity
|
||||
isLocked
|
||||
trending
|
||||
favourites
|
||||
isAdult
|
||||
siteUrl
|
||||
autoCreateForumThread
|
||||
isRecommendationBlocked
|
||||
isReviewBlocked
|
||||
modNotes
|
||||
|
||||
tags {
|
||||
id name description category rank isGeneralSpoiler isMediaSpoiler isAdult userId
|
||||
}
|
||||
|
||||
relations {
|
||||
edges {
|
||||
relationType
|
||||
node { id title { romaji } type format status }
|
||||
}
|
||||
}
|
||||
|
||||
characters(page: 1, perPage: 25, sort: [ROLE, RELEVANCE]) {
|
||||
edges {
|
||||
role
|
||||
name
|
||||
voiceActors(language: JAPANESE, sort: [RELEVANCE, ID]) { id name { full } }
|
||||
node { id name { full } image { large } }
|
||||
}
|
||||
}
|
||||
|
||||
staff(page: 1, perPage: 10, sort: [RELEVANCE, ID]) {
|
||||
edges {
|
||||
role
|
||||
node { id name { full } image { large } }
|
||||
}
|
||||
}
|
||||
|
||||
studios {
|
||||
edges {
|
||||
isMain
|
||||
node { id name isAnimationStudio }
|
||||
}
|
||||
}
|
||||
|
||||
nextAiringEpisode { airingAt timeUntilAiring episode }
|
||||
|
||||
airingSchedule(notYetAired: true, perPage: 1) {
|
||||
nodes { airingAt timeUntilAiring episode }
|
||||
}
|
||||
|
||||
externalLinks {
|
||||
id url site type language color icon notes
|
||||
}
|
||||
|
||||
streamingEpisodes {
|
||||
title thumbnail url site
|
||||
}
|
||||
|
||||
rankings {
|
||||
id rank type format year season allTime context
|
||||
}
|
||||
|
||||
stats {
|
||||
scoreDistribution { score amount }
|
||||
statusDistribution { status amount }
|
||||
}
|
||||
|
||||
recommendations(perPage: 7, sort: RATING_DESC) {
|
||||
nodes {
|
||||
mediaRecommendation {
|
||||
id
|
||||
title { romaji }
|
||||
coverImage { medium }
|
||||
format
|
||||
type
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const BULK_QUERY = `
|
||||
query ($page: Int, $type: MediaType) {
|
||||
Page(page: $page, perPage: 50) {
|
||||
pageInfo { total currentPage lastPage hasNextPage }
|
||||
media(type: $type, sort: ID) {
|
||||
${MEDIA_FIELDS}
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const FEATURED_QUERY = `
|
||||
query ($sort: [MediaSort], $type: MediaType, $status: MediaStatus) {
|
||||
Page(page: 1, perPage: 20) {
|
||||
media(type: $type, sort: $sort, status: $status, isAdult: false) {
|
||||
${MEDIA_FIELDS}
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
// --- NETWORK HELPERS ---
|
||||
async function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function fetchGraphQL(query, variables) {
|
||||
try {
|
||||
const response = await fetch('https://graphql.anilist.co', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' },
|
||||
body: JSON.stringify({ query, variables })
|
||||
});
|
||||
|
||||
const remaining = response.headers.get('X-RateLimit-Remaining');
|
||||
const resetTime = response.headers.get('X-RateLimit-Reset');
|
||||
|
||||
if (remaining && parseInt(remaining) < 10) {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const waitSeconds = resetTime ? (parseInt(resetTime) - now) + 2 : 60;
|
||||
console.warn(`⚠️ Rate Limit Approaching! Sleeping for ${waitSeconds} seconds...`);
|
||||
await sleep(waitSeconds * 1000);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 429) {
|
||||
console.log("Hit absolute rate limit. Sleeping 1 minute...");
|
||||
await sleep(60000);
|
||||
return fetchGraphQL(query, variables);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const json = await response.json();
|
||||
return json.data ? json.data.Page : null;
|
||||
|
||||
} catch (error) {
|
||||
console.error("Fetch Error:", error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// --- FUNCTIONS ---
|
||||
|
||||
function saveMediaBatch(tableName, mediaList) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO ${tableName} (id, title, updatedAt, full_data)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
title=excluded.title, updatedAt=excluded.updatedAt, full_data=excluded.full_data
|
||||
WHERE updatedAt < excluded.updatedAt OR title != excluded.title
|
||||
`);
|
||||
|
||||
db.serialize(() => {
|
||||
db.run("BEGIN TRANSACTION");
|
||||
mediaList.forEach(media => {
|
||||
const title = media.title.english || media.title.romaji || "Unknown";
|
||||
stmt.run(media.id, title, media.updatedAt, JSON.stringify(media));
|
||||
});
|
||||
db.run("COMMIT", (err) => {
|
||||
stmt.finalize();
|
||||
if (err) reject(err); else resolve(mediaList.length);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function updateFeaturedTable(tableName, mediaList) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.serialize(() => {
|
||||
db.run(`DELETE FROM ${tableName}`);
|
||||
const stmt = db.prepare(`INSERT INTO ${tableName} (rank, id, full_data) VALUES (?, ?, ?)`);
|
||||
|
||||
db.run("BEGIN TRANSACTION");
|
||||
mediaList.forEach((media, index) => {
|
||||
stmt.run(index + 1, media.id, JSON.stringify(media));
|
||||
});
|
||||
db.run("COMMIT", (err) => {
|
||||
stmt.finalize();
|
||||
if (err) reject(err); else resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getLocalCount(tableName) {
|
||||
return new Promise((resolve) => db.get(`SELECT COUNT(*) as count FROM ${tableName}`, (err, row) => resolve(row ? row.count : 0)));
|
||||
}
|
||||
|
||||
// --- LOOPS ---
|
||||
|
||||
async function startFeaturedLoop() {
|
||||
console.log(`✨ Starting Featured Content Loop (Refreshes every ${FEATURED_REFRESH_RATE / 60000} mins)`);
|
||||
|
||||
const runUpdate = async () => {
|
||||
console.log("🔄 Refreshing Featured tables (Anime & Books)...");
|
||||
|
||||
// 1. Anime Trending
|
||||
const animeTrending = await fetchGraphQL(FEATURED_QUERY, { sort: "TRENDING_DESC", type: "ANIME" });
|
||||
if (animeTrending && animeTrending.media) {
|
||||
await updateFeaturedTable('trending', animeTrending.media);
|
||||
console.log(` ✅ Updated Anime Trending.`);
|
||||
}
|
||||
|
||||
// 2. Anime Top Airing
|
||||
const animeTop = await fetchGraphQL(FEATURED_QUERY, { sort: "SCORE_DESC", type: "ANIME", status: "RELEASING" });
|
||||
if (animeTop && animeTop.media) {
|
||||
await updateFeaturedTable('top_airing', animeTop.media);
|
||||
console.log(` ✅ Updated Anime Top Airing.`);
|
||||
}
|
||||
|
||||
// 3. Books Trending
|
||||
const mangaTrending = await fetchGraphQL(FEATURED_QUERY, { sort: "TRENDING_DESC", type: "MANGA" });
|
||||
if (mangaTrending && mangaTrending.media) {
|
||||
await updateFeaturedTable('trending_books', mangaTrending.media);
|
||||
console.log(` ✅ Updated Books Trending.`);
|
||||
}
|
||||
|
||||
// 4. Books Popular
|
||||
const mangaPop = await fetchGraphQL(FEATURED_QUERY, { sort: "POPULARITY_DESC", type: "MANGA" });
|
||||
if (mangaPop && mangaPop.media) {
|
||||
await updateFeaturedTable('popular_books', mangaPop.media);
|
||||
console.log(` ✅ Updated Books Popular.`);
|
||||
}
|
||||
};
|
||||
|
||||
await runUpdate();
|
||||
setInterval(runUpdate, FEATURED_REFRESH_RATE);
|
||||
}
|
||||
|
||||
async function startScraper(type, tableName) {
|
||||
let page = 1;
|
||||
let isCaughtUp = false;
|
||||
|
||||
console.log(`🚀 Starting ${type} Scraper (Table: ${tableName})...`);
|
||||
|
||||
while (true) {
|
||||
if (isCaughtUp) {
|
||||
console.log(`💤 ${type} DB caught up. Sleeping 10 mins...`);
|
||||
await sleep(10 * 60 * 1000);
|
||||
console.log(`⏰ Waking up ${type} scraper...`);
|
||||
page = 1;
|
||||
isCaughtUp = false;
|
||||
}
|
||||
|
||||
const data = await fetchGraphQL(BULK_QUERY, { page: page, type: type });
|
||||
|
||||
if (!data || !data.media || data.media.length === 0) {
|
||||
if (data && data.pageInfo && !data.pageInfo.hasNextPage) {
|
||||
console.log(`\n🎉 ${type} Scraper reached the end!`);
|
||||
isCaughtUp = true;
|
||||
} else {
|
||||
await sleep(5000);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
await saveMediaBatch(tableName, data.media);
|
||||
const totalInDb = await getLocalCount(tableName);
|
||||
const percent = data.pageInfo.total ? ((page * 50 / data.pageInfo.total) * 100).toFixed(2) : "??";
|
||||
|
||||
process.stdout.write(`\r📥 ${type}: Page ${data.pageInfo.currentPage} | DB Total: ${totalInDb} | ~${percent}%`);
|
||||
|
||||
if (data.pageInfo.hasNextPage) {
|
||||
page++;
|
||||
await sleep(DELAY_MS);
|
||||
} else {
|
||||
console.log(`\n🎉 ${type} Scraper reached the end!`);
|
||||
isCaughtUp = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- MAIN ENTRY ---
|
||||
async function animeMetadata() {
|
||||
await initDB();
|
||||
|
||||
// Start loops
|
||||
startFeaturedLoop();
|
||||
startScraper('ANIME', 'anime');
|
||||
startScraper('MANGA', 'books');
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
animeMetadata();
|
||||
}
|
||||
|
||||
module.exports = { animeMetadata };
|
||||
BIN
src/metadata/anilist_anime.db
Normal file
BIN
src/metadata/anilist_anime.db
Normal file
Binary file not shown.
Reference in New Issue
Block a user