code organisation & refactor
This commit is contained in:
@@ -2,13 +2,11 @@ const sqlite3 = require('sqlite3').verbose();
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
// --- CONFIGURATION ---
|
||||
const DB_PATH = path.join(__dirname, 'anilist_anime.db');
|
||||
const REQUESTS_PER_MINUTE = 20; // 20 RPM is safe (AniList limit is 90)
|
||||
const REQUESTS_PER_MINUTE = 20;
|
||||
const DELAY_MS = (60000 / REQUESTS_PER_MINUTE);
|
||||
const FEATURED_REFRESH_RATE = 8 * 60 * 1000; // 8 Minutes
|
||||
const FEATURED_REFRESH_RATE = 8 * 60 * 1000;
|
||||
|
||||
// Ensure directory exists
|
||||
const dir = path.dirname(DB_PATH);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
@@ -16,16 +14,13 @@ if (!fs.existsSync(dir)) {
|
||||
|
||||
const db = new sqlite3.Database(DB_PATH);
|
||||
|
||||
// --- DATABASE SETUP ---
|
||||
function initDB() {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.serialize(() => {
|
||||
// 1. Anime Tables
|
||||
db.run(`CREATE TABLE IF NOT EXISTS anime (id INTEGER PRIMARY KEY, title TEXT, updatedAt INTEGER, full_data JSON)`);
|
||||
db.run(`CREATE TABLE IF NOT EXISTS trending (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
|
||||
db.run(`CREATE TABLE IF NOT EXISTS top_airing (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
|
||||
|
||||
// 2. Books Tables (Manga/LN)
|
||||
db.run(`CREATE TABLE IF NOT EXISTS books (id INTEGER PRIMARY KEY, title TEXT, updatedAt INTEGER, full_data JSON)`);
|
||||
db.run(`CREATE TABLE IF NOT EXISTS trending_books (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
|
||||
db.run(`CREATE TABLE IF NOT EXISTS popular_books (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`, (err) => {
|
||||
@@ -36,9 +31,7 @@ function initDB() {
|
||||
});
|
||||
}
|
||||
|
||||
// --- QUERIES ---
|
||||
|
||||
// Exhaustive list of fields
|
||||
const MEDIA_FIELDS = `
|
||||
id
|
||||
idMal
|
||||
@@ -170,7 +163,6 @@ query ($sort: [MediaSort], $type: MediaType, $status: MediaStatus) {
|
||||
}
|
||||
`;
|
||||
|
||||
// --- NETWORK HELPERS ---
|
||||
async function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -211,7 +203,6 @@ async function fetchGraphQL(query, variables) {
|
||||
}
|
||||
}
|
||||
|
||||
// --- FUNCTIONS ---
|
||||
|
||||
function saveMediaBatch(tableName, mediaList) {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -259,7 +250,6 @@ function getLocalCount(tableName) {
|
||||
return new Promise((resolve) => db.get(`SELECT COUNT(*) as count FROM ${tableName}`, (err, row) => resolve(row ? row.count : 0)));
|
||||
}
|
||||
|
||||
// --- LOOPS ---
|
||||
|
||||
async function startFeaturedLoop() {
|
||||
console.log(`✨ Starting Featured Content Loop (Refreshes every ${FEATURED_REFRESH_RATE / 60000} mins)`);
|
||||
@@ -267,28 +257,24 @@ async function startFeaturedLoop() {
|
||||
const runUpdate = async () => {
|
||||
console.log("🔄 Refreshing Featured tables (Anime & Books)...");
|
||||
|
||||
// 1. Anime Trending
|
||||
const animeTrending = await fetchGraphQL(FEATURED_QUERY, { sort: "TRENDING_DESC", type: "ANIME" });
|
||||
if (animeTrending && animeTrending.media) {
|
||||
await updateFeaturedTable('trending', animeTrending.media);
|
||||
console.log(` ✅ Updated Anime Trending.`);
|
||||
}
|
||||
|
||||
// 2. Anime Top Airing
|
||||
const animeTop = await fetchGraphQL(FEATURED_QUERY, { sort: "SCORE_DESC", type: "ANIME", status: "RELEASING" });
|
||||
if (animeTop && animeTop.media) {
|
||||
await updateFeaturedTable('top_airing', animeTop.media);
|
||||
console.log(` ✅ Updated Anime Top Airing.`);
|
||||
}
|
||||
|
||||
// 3. Books Trending
|
||||
const mangaTrending = await fetchGraphQL(FEATURED_QUERY, { sort: "TRENDING_DESC", type: "MANGA" });
|
||||
if (mangaTrending && mangaTrending.media) {
|
||||
await updateFeaturedTable('trending_books', mangaTrending.media);
|
||||
console.log(` ✅ Updated Books Trending.`);
|
||||
}
|
||||
|
||||
// 4. Books Popular
|
||||
const mangaPop = await fetchGraphQL(FEATURED_QUERY, { sort: "POPULARITY_DESC", type: "MANGA" });
|
||||
if (mangaPop && mangaPop.media) {
|
||||
await updateFeaturedTable('popular_books', mangaPop.media);
|
||||
@@ -343,12 +329,10 @@ async function startScraper(type, tableName) {
|
||||
}
|
||||
}
|
||||
|
||||
// --- MAIN ENTRY ---
|
||||
async function animeMetadata() {
|
||||
await initDB();
|
||||
|
||||
// Start loops
|
||||
startFeaturedLoop();
|
||||
startFeaturedLoop();
|
||||
startScraper('ANIME', 'anime');
|
||||
startScraper('MANGA', 'books');
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user