code organisation & refactor

This commit is contained in:
2025-11-27 20:23:00 +01:00
parent 03636fee99
commit 2b29beeeb5
33 changed files with 2099 additions and 1947 deletions

View File

@@ -2,13 +2,11 @@ const sqlite3 = require('sqlite3').verbose();
const path = require('path');
const fs = require('fs');
// --- CONFIGURATION ---
const DB_PATH = path.join(__dirname, 'anilist_anime.db');
const REQUESTS_PER_MINUTE = 20; // 20 RPM is safe (AniList limit is 90)
const REQUESTS_PER_MINUTE = 20;
const DELAY_MS = (60000 / REQUESTS_PER_MINUTE);
const FEATURED_REFRESH_RATE = 8 * 60 * 1000; // 8 Minutes
const FEATURED_REFRESH_RATE = 8 * 60 * 1000;
// Ensure directory exists
const dir = path.dirname(DB_PATH);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
@@ -16,16 +14,13 @@ if (!fs.existsSync(dir)) {
const db = new sqlite3.Database(DB_PATH);
// --- DATABASE SETUP ---
function initDB() {
return new Promise((resolve, reject) => {
db.serialize(() => {
// 1. Anime Tables
db.run(`CREATE TABLE IF NOT EXISTS anime (id INTEGER PRIMARY KEY, title TEXT, updatedAt INTEGER, full_data JSON)`);
db.run(`CREATE TABLE IF NOT EXISTS trending (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
db.run(`CREATE TABLE IF NOT EXISTS top_airing (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
// 2. Books Tables (Manga/LN)
db.run(`CREATE TABLE IF NOT EXISTS books (id INTEGER PRIMARY KEY, title TEXT, updatedAt INTEGER, full_data JSON)`);
db.run(`CREATE TABLE IF NOT EXISTS trending_books (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
db.run(`CREATE TABLE IF NOT EXISTS popular_books (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`, (err) => {
@@ -36,9 +31,7 @@ function initDB() {
});
}
// --- QUERIES ---
// Exhaustive list of fields
const MEDIA_FIELDS = `
id
idMal
@@ -170,7 +163,6 @@ query ($sort: [MediaSort], $type: MediaType, $status: MediaStatus) {
}
`;
// --- NETWORK HELPERS ---
async function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
@@ -211,7 +203,6 @@ async function fetchGraphQL(query, variables) {
}
}
// --- FUNCTIONS ---
function saveMediaBatch(tableName, mediaList) {
return new Promise((resolve, reject) => {
@@ -259,7 +250,6 @@ function getLocalCount(tableName) {
return new Promise((resolve) => db.get(`SELECT COUNT(*) as count FROM ${tableName}`, (err, row) => resolve(row ? row.count : 0)));
}
// --- LOOPS ---
async function startFeaturedLoop() {
console.log(`✨ Starting Featured Content Loop (Refreshes every ${FEATURED_REFRESH_RATE / 60000} mins)`);
@@ -267,28 +257,24 @@ async function startFeaturedLoop() {
const runUpdate = async () => {
console.log("🔄 Refreshing Featured tables (Anime & Books)...");
// 1. Anime Trending
const animeTrending = await fetchGraphQL(FEATURED_QUERY, { sort: "TRENDING_DESC", type: "ANIME" });
if (animeTrending && animeTrending.media) {
await updateFeaturedTable('trending', animeTrending.media);
console.log(` ✅ Updated Anime Trending.`);
}
// 2. Anime Top Airing
const animeTop = await fetchGraphQL(FEATURED_QUERY, { sort: "SCORE_DESC", type: "ANIME", status: "RELEASING" });
if (animeTop && animeTop.media) {
await updateFeaturedTable('top_airing', animeTop.media);
console.log(` ✅ Updated Anime Top Airing.`);
}
// 3. Books Trending
const mangaTrending = await fetchGraphQL(FEATURED_QUERY, { sort: "TRENDING_DESC", type: "MANGA" });
if (mangaTrending && mangaTrending.media) {
await updateFeaturedTable('trending_books', mangaTrending.media);
console.log(` ✅ Updated Books Trending.`);
}
// 4. Books Popular
const mangaPop = await fetchGraphQL(FEATURED_QUERY, { sort: "POPULARITY_DESC", type: "MANGA" });
if (mangaPop && mangaPop.media) {
await updateFeaturedTable('popular_books', mangaPop.media);
@@ -343,12 +329,10 @@ async function startScraper(type, tableName) {
}
}
// --- MAIN ENTRY ---
async function animeMetadata() {
await initDB();
// Start loops
startFeaturedLoop();
startFeaturedLoop();
startScraper('ANIME', 'anime');
startScraper('MANGA', 'books');
}