organisation & minor fixes
This commit is contained in:
10
server.js
10
server.js
@@ -7,11 +7,11 @@ const { initDatabase } = require('./src/shared/database');
|
|||||||
const { loadExtensions } = require('./src/shared/extensions');
|
const { loadExtensions } = require('./src/shared/extensions');
|
||||||
|
|
||||||
const viewsRoutes = require('./src/views/views.routes');
|
const viewsRoutes = require('./src/views/views.routes');
|
||||||
const animeRoutes = require('./src/anime/anime.routes');
|
const animeRoutes = require('./src/api/anime/anime.routes');
|
||||||
const booksRoutes = require('./src/books/books.routes');
|
const booksRoutes = require('./src/api/books/books.routes');
|
||||||
const proxyRoutes = require('./src/shared/proxy/proxy.routes');
|
const proxyRoutes = require('./src/api/proxy/proxy.routes');
|
||||||
const extensionsRoutes = require('./src/extensions/extensions.routes');
|
const extensionsRoutes = require('./src/api/extensions/extensions.routes');
|
||||||
const galleryRoutes = require('./src/gallery/gallery.routes');
|
const galleryRoutes = require('./src/api/gallery/gallery.routes');
|
||||||
|
|
||||||
fastify.register(require('@fastify/static'), {
|
fastify.register(require('@fastify/static'), {
|
||||||
root: path.join(__dirname, 'public'),
|
root: path.join(__dirname, 'public'),
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import {FastifyReply, FastifyRequest} from 'fastify';
|
import {FastifyReply, FastifyRequest} from 'fastify';
|
||||||
import * as animeService from './anime.service';
|
import * as animeService from './anime.service';
|
||||||
import {getExtension} from '../shared/extensions';
|
import {getExtension} from '../../shared/extensions';
|
||||||
import {Anime, AnimeRequest, SearchRequest, WatchStreamRequest} from '../types';
|
import {Anime, AnimeRequest, SearchRequest, WatchStreamRequest} from '../types';
|
||||||
|
|
||||||
export async function getAnime(req: AnimeRequest, reply: FastifyReply) {
|
export async function getAnime(req: AnimeRequest, reply: FastifyReply) {
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import {queryOne, queryAll, getCache, setCache, getCachedExtension, cacheExtension, getExtensionTitle } from '../shared/database';
|
import { getCache, setCache, getCachedExtension, cacheExtension, getExtensionTitle } from '../../shared/queries';
|
||||||
|
import { queryAll, queryOne } from '../../shared/database';
|
||||||
import {Anime, Episode, Extension, StreamData} from '../types';
|
import {Anime, Episode, Extension, StreamData} from '../types';
|
||||||
|
|
||||||
const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
import {FastifyReply, FastifyRequest} from 'fastify';
|
import {FastifyReply, FastifyRequest} from 'fastify';
|
||||||
import * as booksService from './books.service';
|
import * as booksService from './books.service';
|
||||||
import {getExtension} from '../shared/extensions';
|
import {getExtension} from '../../shared/extensions';
|
||||||
import {BookRequest, ChapterRequest, SearchRequest} from '../types';
|
import {BookRequest, ChapterRequest, SearchRequest} from '../types';
|
||||||
|
|
||||||
export async function getBook(req: BookRequest, reply: FastifyReply) {
|
export async function getBook(req: BookRequest, reply: FastifyReply) {
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { queryOne, queryAll, getCachedExtension, cacheExtension, getCache, setCache, getExtensionTitle } from '../shared/database';
|
import { getCachedExtension, cacheExtension, getCache, setCache, getExtensionTitle } from '../../shared/queries';
|
||||||
import { getAllExtensions, getBookExtensionsMap } from '../shared/extensions';
|
import { queryOne, queryAll } from '../../shared/database';
|
||||||
|
import { getAllExtensions, getBookExtensionsMap } from '../../shared/extensions';
|
||||||
import { Book, Extension, ChapterWithProvider, ChapterContent } from '../types';
|
import { Book, Extension, ChapterWithProvider, ChapterContent } from '../types';
|
||||||
|
|
||||||
const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||||
import { getExtension, getExtensionsList, getGalleryExtensionsMap, getBookExtensionsMap, getAnimeExtensionsMap } from '../shared/extensions';
|
import { getExtension, getExtensionsList, getGalleryExtensionsMap, getBookExtensionsMap, getAnimeExtensionsMap } from '../../shared/extensions';
|
||||||
import { ExtensionNameRequest } from '../types';
|
import { ExtensionNameRequest } from '../types';
|
||||||
|
|
||||||
export async function getExtensions(req: FastifyRequest, reply: FastifyReply) {
|
export async function getExtensions(req: FastifyRequest, reply: FastifyReply) {
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
import { getAllExtensions, getExtension } from '../shared/extensions';
|
import { getAllExtensions, getExtension } from '../../shared/extensions';
|
||||||
import { GallerySearchResult, GalleryInfo, Favorite, FavoriteResult } from '../types';
|
import { GallerySearchResult, GalleryInfo, Favorite, FavoriteResult } from '../types';
|
||||||
import { getDatabase } from '../shared/database';
|
import { getDatabase } from '../../shared/database';
|
||||||
|
|
||||||
export async function searchGallery(query: string, page: number = 1, perPage: number = 48): Promise<GallerySearchResult> {
|
export async function searchGallery(query: string, page: number = 1, perPage: number = 48): Promise<GallerySearchResult> {
|
||||||
const extensions = getAllExtensions();
|
const extensions = getAllExtensions();
|
||||||
45
src/api/proxy/proxy.controller.ts
Normal file
45
src/api/proxy/proxy.controller.ts
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
import { FastifyReply } from 'fastify';
|
||||||
|
import { proxyRequest, processM3U8Content, streamToReadable } from './proxy.service';
|
||||||
|
import { ProxyRequest } from '../types';
|
||||||
|
|
||||||
|
export async function handleProxy(req: ProxyRequest, reply: FastifyReply) {
|
||||||
|
const { url, referer, origin, userAgent } = req.query;
|
||||||
|
|
||||||
|
if (!url) {
|
||||||
|
return reply.code(400).send({ error: "No URL provided" });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { response, contentType, isM3U8 } = await proxyRequest(url, {
|
||||||
|
referer,
|
||||||
|
origin,
|
||||||
|
userAgent
|
||||||
|
});
|
||||||
|
|
||||||
|
reply.header('Access-Control-Allow-Origin', '*');
|
||||||
|
reply.header('Access-Control-Allow-Methods', 'GET, OPTIONS');
|
||||||
|
|
||||||
|
if (contentType) {
|
||||||
|
reply.header('Content-Type', contentType);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isM3U8) {
|
||||||
|
const text = await response.text();
|
||||||
|
const baseUrl = new URL(response.url);
|
||||||
|
|
||||||
|
const processed = processM3U8Content(text, baseUrl, {
|
||||||
|
referer,
|
||||||
|
origin,
|
||||||
|
userAgent
|
||||||
|
});
|
||||||
|
|
||||||
|
return processed;
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send(streamToReadable(response.body!));
|
||||||
|
|
||||||
|
} catch (err) {
|
||||||
|
req.server.log.error(err);
|
||||||
|
return reply.code(500).send({ error: "Internal Server Error" });
|
||||||
|
}
|
||||||
|
}
|
||||||
8
src/api/proxy/proxy.routes.ts
Normal file
8
src/api/proxy/proxy.routes.ts
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
import { FastifyInstance } from 'fastify';
|
||||||
|
import { handleProxy } from './proxy.controller';
|
||||||
|
|
||||||
|
async function proxyRoutes(fastify: FastifyInstance) {
|
||||||
|
fastify.get('/proxy', handleProxy);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default proxyRoutes;
|
||||||
@@ -1,344 +0,0 @@
|
|||||||
const sqlite3 = require('sqlite3').verbose();
|
|
||||||
const path = require('path');
|
|
||||||
const fs = require('fs');
|
|
||||||
|
|
||||||
const DB_PATH = path.join(__dirname, 'anilist_anime.db');
|
|
||||||
const REQUESTS_PER_MINUTE = 20;
|
|
||||||
const DELAY_MS = (60000 / REQUESTS_PER_MINUTE);
|
|
||||||
const FEATURED_REFRESH_RATE = 8 * 60 * 1000;
|
|
||||||
|
|
||||||
const dir = path.dirname(DB_PATH);
|
|
||||||
if (!fs.existsSync(dir)) {
|
|
||||||
fs.mkdirSync(dir, { recursive: true });
|
|
||||||
}
|
|
||||||
|
|
||||||
const db = new sqlite3.Database(DB_PATH);
|
|
||||||
|
|
||||||
function initDB() {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
db.serialize(() => {
|
|
||||||
db.run(`CREATE TABLE IF NOT EXISTS anime (id INTEGER PRIMARY KEY, title TEXT, updatedAt INTEGER, full_data JSON)`);
|
|
||||||
db.run(`CREATE TABLE IF NOT EXISTS trending (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
|
|
||||||
db.run(`CREATE TABLE IF NOT EXISTS top_airing (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
|
|
||||||
|
|
||||||
db.run(`CREATE TABLE IF NOT EXISTS books (id INTEGER PRIMARY KEY, title TEXT, updatedAt INTEGER, full_data JSON)`);
|
|
||||||
db.run(`CREATE TABLE IF NOT EXISTS trending_books (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`);
|
|
||||||
db.run(`CREATE TABLE IF NOT EXISTS popular_books (rank INTEGER PRIMARY KEY, id INTEGER, full_data JSON)`, (err) => {
|
|
||||||
if (err) reject(err);
|
|
||||||
else resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
const MEDIA_FIELDS = `
|
|
||||||
id
|
|
||||||
idMal
|
|
||||||
title { romaji english native userPreferred }
|
|
||||||
type
|
|
||||||
format
|
|
||||||
status
|
|
||||||
description
|
|
||||||
startDate { year month day }
|
|
||||||
endDate { year month day }
|
|
||||||
season
|
|
||||||
seasonYear
|
|
||||||
seasonInt
|
|
||||||
episodes
|
|
||||||
duration
|
|
||||||
chapters
|
|
||||||
volumes
|
|
||||||
countryOfOrigin
|
|
||||||
isLicensed
|
|
||||||
source
|
|
||||||
hashtag
|
|
||||||
trailer { id site thumbnail }
|
|
||||||
updatedAt
|
|
||||||
coverImage { extraLarge large medium color }
|
|
||||||
bannerImage
|
|
||||||
genres
|
|
||||||
synonyms
|
|
||||||
averageScore
|
|
||||||
meanScore
|
|
||||||
popularity
|
|
||||||
isLocked
|
|
||||||
trending
|
|
||||||
favourites
|
|
||||||
isAdult
|
|
||||||
siteUrl
|
|
||||||
autoCreateForumThread
|
|
||||||
isRecommendationBlocked
|
|
||||||
isReviewBlocked
|
|
||||||
modNotes
|
|
||||||
|
|
||||||
tags {
|
|
||||||
id name description category rank isGeneralSpoiler isMediaSpoiler isAdult userId
|
|
||||||
}
|
|
||||||
|
|
||||||
relations {
|
|
||||||
edges {
|
|
||||||
relationType
|
|
||||||
node { id title { romaji } type format status }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
characters(page: 1, perPage: 25, sort: [ROLE, RELEVANCE]) {
|
|
||||||
edges {
|
|
||||||
role
|
|
||||||
name
|
|
||||||
voiceActors(language: JAPANESE, sort: [RELEVANCE, ID]) { id name { full } }
|
|
||||||
node { id name { full } image { large } }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
staff(page: 1, perPage: 10, sort: [RELEVANCE, ID]) {
|
|
||||||
edges {
|
|
||||||
role
|
|
||||||
node { id name { full } image { large } }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
studios {
|
|
||||||
edges {
|
|
||||||
isMain
|
|
||||||
node { id name isAnimationStudio }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
nextAiringEpisode { airingAt timeUntilAiring episode }
|
|
||||||
|
|
||||||
airingSchedule(notYetAired: true, perPage: 1) {
|
|
||||||
nodes { airingAt timeUntilAiring episode }
|
|
||||||
}
|
|
||||||
|
|
||||||
externalLinks {
|
|
||||||
id url site type language color icon notes
|
|
||||||
}
|
|
||||||
|
|
||||||
streamingEpisodes {
|
|
||||||
title thumbnail url site
|
|
||||||
}
|
|
||||||
|
|
||||||
rankings {
|
|
||||||
id rank type format year season allTime context
|
|
||||||
}
|
|
||||||
|
|
||||||
stats {
|
|
||||||
scoreDistribution { score amount }
|
|
||||||
statusDistribution { status amount }
|
|
||||||
}
|
|
||||||
|
|
||||||
recommendations(perPage: 7, sort: RATING_DESC) {
|
|
||||||
nodes {
|
|
||||||
mediaRecommendation {
|
|
||||||
id
|
|
||||||
title { romaji }
|
|
||||||
coverImage { medium }
|
|
||||||
format
|
|
||||||
type
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
`;
|
|
||||||
|
|
||||||
const BULK_QUERY = `
|
|
||||||
query ($page: Int, $type: MediaType) {
|
|
||||||
Page(page: $page, perPage: 50) {
|
|
||||||
pageInfo { total currentPage lastPage hasNextPage }
|
|
||||||
media(type: $type, sort: ID) {
|
|
||||||
${MEDIA_FIELDS}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
`;
|
|
||||||
|
|
||||||
const FEATURED_QUERY = `
|
|
||||||
query ($sort: [MediaSort], $type: MediaType, $status: MediaStatus) {
|
|
||||||
Page(page: 1, perPage: 20) {
|
|
||||||
media(type: $type, sort: $sort, status: $status, isAdult: false) {
|
|
||||||
${MEDIA_FIELDS}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
`;
|
|
||||||
|
|
||||||
async function sleep(ms) {
|
|
||||||
return new Promise(resolve => setTimeout(resolve, ms));
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchGraphQL(query, variables) {
|
|
||||||
try {
|
|
||||||
const response = await fetch('https://graphql.anilist.co', {
|
|
||||||
method: 'POST',
|
|
||||||
headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' },
|
|
||||||
body: JSON.stringify({ query, variables })
|
|
||||||
});
|
|
||||||
|
|
||||||
const remaining = response.headers.get('X-RateLimit-Remaining');
|
|
||||||
const resetTime = response.headers.get('X-RateLimit-Reset');
|
|
||||||
|
|
||||||
if (remaining && parseInt(remaining) < 10) {
|
|
||||||
const now = Math.floor(Date.now() / 1000);
|
|
||||||
const waitSeconds = resetTime ? (parseInt(resetTime) - now) + 2 : 60;
|
|
||||||
console.warn(`⚠️ Rate Limit Approaching! Sleeping for ${waitSeconds} seconds...`);
|
|
||||||
await sleep(waitSeconds * 1000);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
if (response.status === 429) {
|
|
||||||
console.log("Hit absolute rate limit. Sleeping 1 minute...");
|
|
||||||
await sleep(60000);
|
|
||||||
return fetchGraphQL(query, variables);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const json = await response.json();
|
|
||||||
return json.data ? json.data.Page : null;
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error("Fetch Error:", error.message);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function saveMediaBatch(tableName, mediaList) {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const stmt = db.prepare(`
|
|
||||||
INSERT INTO ${tableName} (id, title, updatedAt, full_data)
|
|
||||||
VALUES (?, ?, ?, ?)
|
|
||||||
ON CONFLICT(id) DO UPDATE SET
|
|
||||||
title=excluded.title, updatedAt=excluded.updatedAt, full_data=excluded.full_data
|
|
||||||
WHERE updatedAt < excluded.updatedAt OR title != excluded.title
|
|
||||||
`);
|
|
||||||
|
|
||||||
db.serialize(() => {
|
|
||||||
db.run("BEGIN TRANSACTION");
|
|
||||||
mediaList.forEach(media => {
|
|
||||||
const title = media.title.english || media.title.romaji || "Unknown";
|
|
||||||
stmt.run(media.id, title, media.updatedAt, JSON.stringify(media));
|
|
||||||
});
|
|
||||||
db.run("COMMIT", (err) => {
|
|
||||||
stmt.finalize();
|
|
||||||
if (err) reject(err); else resolve(mediaList.length);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function updateFeaturedTable(tableName, mediaList) {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
db.serialize(() => {
|
|
||||||
db.run(`DELETE FROM ${tableName}`);
|
|
||||||
const stmt = db.prepare(`INSERT INTO ${tableName} (rank, id, full_data) VALUES (?, ?, ?)`);
|
|
||||||
|
|
||||||
db.run("BEGIN TRANSACTION");
|
|
||||||
mediaList.forEach((media, index) => {
|
|
||||||
stmt.run(index + 1, media.id, JSON.stringify(media));
|
|
||||||
});
|
|
||||||
db.run("COMMIT", (err) => {
|
|
||||||
stmt.finalize();
|
|
||||||
if (err) reject(err); else resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function getLocalCount(tableName) {
|
|
||||||
return new Promise((resolve) => db.get(`SELECT COUNT(*) as count FROM ${tableName}`, (err, row) => resolve(row ? row.count : 0)));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async function startFeaturedLoop() {
|
|
||||||
console.log(`✨ Starting Featured Content Loop (Refreshes every ${FEATURED_REFRESH_RATE / 60000} mins)`);
|
|
||||||
|
|
||||||
const runUpdate = async () => {
|
|
||||||
console.log("🔄 Refreshing Featured tables (Anime & Books)...");
|
|
||||||
|
|
||||||
const animeTrending = await fetchGraphQL(FEATURED_QUERY, { sort: "TRENDING_DESC", type: "ANIME" });
|
|
||||||
if (animeTrending && animeTrending.media) {
|
|
||||||
await updateFeaturedTable('trending', animeTrending.media);
|
|
||||||
console.log(` ✅ Updated Anime Trending.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const animeTop = await fetchGraphQL(FEATURED_QUERY, { sort: "SCORE_DESC", type: "ANIME", status: "RELEASING" });
|
|
||||||
if (animeTop && animeTop.media) {
|
|
||||||
await updateFeaturedTable('top_airing', animeTop.media);
|
|
||||||
console.log(` ✅ Updated Anime Top Airing.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const mangaTrending = await fetchGraphQL(FEATURED_QUERY, { sort: "TRENDING_DESC", type: "MANGA" });
|
|
||||||
if (mangaTrending && mangaTrending.media) {
|
|
||||||
await updateFeaturedTable('trending_books', mangaTrending.media);
|
|
||||||
console.log(` ✅ Updated Books Trending.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const mangaPop = await fetchGraphQL(FEATURED_QUERY, { sort: "POPULARITY_DESC", type: "MANGA" });
|
|
||||||
if (mangaPop && mangaPop.media) {
|
|
||||||
await updateFeaturedTable('popular_books', mangaPop.media);
|
|
||||||
console.log(` ✅ Updated Books Popular.`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
await runUpdate();
|
|
||||||
setInterval(runUpdate, FEATURED_REFRESH_RATE);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function startScraper(type, tableName) {
|
|
||||||
let page = 1;
|
|
||||||
let isCaughtUp = false;
|
|
||||||
|
|
||||||
console.log(`🚀 Starting ${type} Scraper (Table: ${tableName})...`);
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
if (isCaughtUp) {
|
|
||||||
console.log(`💤 ${type} DB caught up. Sleeping 10 mins...`);
|
|
||||||
await sleep(10 * 60 * 1000);
|
|
||||||
console.log(`⏰ Waking up ${type} scraper...`);
|
|
||||||
page = 1;
|
|
||||||
isCaughtUp = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await fetchGraphQL(BULK_QUERY, { page: page, type: type });
|
|
||||||
|
|
||||||
if (!data || !data.media || data.media.length === 0) {
|
|
||||||
if (data && data.pageInfo && !data.pageInfo.hasNextPage) {
|
|
||||||
console.log(`\n🎉 ${type} Scraper reached the end!`);
|
|
||||||
isCaughtUp = true;
|
|
||||||
} else {
|
|
||||||
await sleep(5000);
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
await saveMediaBatch(tableName, data.media);
|
|
||||||
const totalInDb = await getLocalCount(tableName);
|
|
||||||
const percent = data.pageInfo.total ? ((page * 50 / data.pageInfo.total) * 100).toFixed(2) : "??";
|
|
||||||
|
|
||||||
process.stdout.write(`\r📥 ${type}: Page ${data.pageInfo.currentPage} | DB Total: ${totalInDb} | ~${percent}%`);
|
|
||||||
|
|
||||||
if (data.pageInfo.hasNextPage) {
|
|
||||||
page++;
|
|
||||||
await sleep(DELAY_MS);
|
|
||||||
} else {
|
|
||||||
console.log(`\n🎉 ${type} Scraper reached the end!`);
|
|
||||||
isCaughtUp = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function animeMetadata() {
|
|
||||||
await initDB();
|
|
||||||
|
|
||||||
startFeaturedLoop();
|
|
||||||
startScraper('ANIME', 'anime');
|
|
||||||
startScraper('MANGA', 'books');
|
|
||||||
}
|
|
||||||
|
|
||||||
if (require.main === module) {
|
|
||||||
animeMetadata();
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = { animeMetadata };
|
|
||||||
@@ -141,7 +141,7 @@ async function toggleFavorite(item) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function createGalleryCard(item) {
|
function createGalleryCard(item, isLoadMore = false) {
|
||||||
const card = document.createElement('a');
|
const card = document.createElement('a');
|
||||||
card.className = 'gallery-card grid-item';
|
card.className = 'gallery-card grid-item';
|
||||||
card.href = `/gallery/${item.provider || currentProvider || 'unknown'}/${encodeURIComponent(item.id)}`;
|
card.href = `/gallery/${item.provider || currentProvider || 'unknown'}/${encodeURIComponent(item.id)}`;
|
||||||
@@ -151,9 +151,10 @@ function createGalleryCard(item) {
|
|||||||
img.className = 'gallery-card-img';
|
img.className = 'gallery-card-img';
|
||||||
|
|
||||||
img.src = getProxiedImageUrl(item);
|
img.src = getProxiedImageUrl(item);
|
||||||
|
|
||||||
img.alt = getTagsArray(item).join(', ') || 'Image';
|
img.alt = getTagsArray(item).join(', ') || 'Image';
|
||||||
img.loading = 'lazy';
|
if (isLoadMore) {
|
||||||
|
img.loading = 'lazy';
|
||||||
|
}
|
||||||
|
|
||||||
const favBtn = document.createElement('button');
|
const favBtn = document.createElement('button');
|
||||||
favBtn.className = 'fav-btn';
|
favBtn.className = 'fav-btn';
|
||||||
@@ -273,7 +274,7 @@ async function searchGallery(isLoadMore = false) {
|
|||||||
if (msnry) msnry.remove(toRemove);
|
if (msnry) msnry.remove(toRemove);
|
||||||
toRemove.forEach(el => el.remove());
|
toRemove.forEach(el => el.remove());
|
||||||
|
|
||||||
const newCards = results.map(item => createGalleryCard(item));
|
const newCards = results.map((item, index) => createGalleryCard(item, isLoadMore));
|
||||||
newCards.forEach(card => resultsContainer.appendChild(card));
|
newCards.forEach(card => resultsContainer.appendChild(card));
|
||||||
if (msnry) msnry.appended(newCards);
|
if (msnry) msnry.appended(newCards);
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ const databases = new Map();
|
|||||||
const DEFAULT_PATHS = {
|
const DEFAULT_PATHS = {
|
||||||
anilist: path.join(__dirname, '..', 'metadata', 'anilist_anime.db'),
|
anilist: path.join(__dirname, '..', 'metadata', 'anilist_anime.db'),
|
||||||
favorites: path.join(os.homedir(), "WaifuBoards", "favorites.db"),
|
favorites: path.join(os.homedir(), "WaifuBoards", "favorites.db"),
|
||||||
|
|
||||||
cache: path.join(os.homedir(), "WaifuBoards", "cache.db")
|
cache: path.join(os.homedir(), "WaifuBoards", "cache.db")
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -21,7 +20,6 @@ async function ensureExtensionsTable(db) {
|
|||||||
title TEXT NOT NULL,
|
title TEXT NOT NULL,
|
||||||
metadata TEXT NOT NULL,
|
metadata TEXT NOT NULL,
|
||||||
updated_at INTEGER NOT NULL,
|
updated_at INTEGER NOT NULL,
|
||||||
|
|
||||||
PRIMARY KEY(ext_name, id)
|
PRIMARY KEY(ext_name, id)
|
||||||
);
|
);
|
||||||
`, (err) => {
|
`, (err) => {
|
||||||
@@ -157,7 +155,6 @@ function initDatabase(name = 'anilist', dbPath = null, readOnly = false) {
|
|||||||
|
|
||||||
function getDatabase(name = 'anilist') {
|
function getDatabase(name = 'anilist') {
|
||||||
if (!databases.has(name)) {
|
if (!databases.has(name)) {
|
||||||
|
|
||||||
const readOnly = (name === 'anilist');
|
const readOnly = (name === 'anilist');
|
||||||
return initDatabase(name, null, readOnly);
|
return initDatabase(name, null, readOnly);
|
||||||
}
|
}
|
||||||
@@ -208,73 +205,11 @@ function closeDatabase(name = null) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getCachedExtension(extName, id) {
|
|
||||||
return queryOne(
|
|
||||||
"SELECT metadata FROM extension WHERE ext_name = ? AND id = ?",
|
|
||||||
[extName, id]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function cacheExtension(extName, id, title, metadata) {
|
|
||||||
return run(
|
|
||||||
`
|
|
||||||
INSERT INTO extension (ext_name, id, title, metadata, updated_at)
|
|
||||||
VALUES (?, ?, ?, ?, ?)
|
|
||||||
ON CONFLICT(ext_name, id)
|
|
||||||
DO UPDATE SET
|
|
||||||
title = excluded.title,
|
|
||||||
metadata = excluded.metadata,
|
|
||||||
updated_at = ?
|
|
||||||
`,
|
|
||||||
[extName, id, title, JSON.stringify(metadata), Date.now(), Date.now()]
|
|
||||||
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getExtensionTitle(extName, id) {
|
|
||||||
|
|
||||||
const sql = "SELECT title FROM extension WHERE ext_name = ? AND id = ?";
|
|
||||||
|
|
||||||
const row = await queryOne(sql, [extName, id], 'anilist');
|
|
||||||
|
|
||||||
return row ? row.title : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function deleteExtension(extName) {
|
|
||||||
return run(
|
|
||||||
"DELETE FROM extension WHERE ext_name = ?",
|
|
||||||
[extName]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getCache(key) {
|
|
||||||
return queryOne("SELECT result, created_at, ttl_ms FROM cache WHERE key = ?", [key], "cache");
|
|
||||||
}
|
|
||||||
|
|
||||||
async function setCache(key, result, ttl_ms) {
|
|
||||||
return run(
|
|
||||||
`
|
|
||||||
INSERT INTO cache (key, result, created_at, ttl_ms)
|
|
||||||
VALUES (?, ?, ?, ?)
|
|
||||||
ON CONFLICT(key)
|
|
||||||
DO UPDATE SET result = excluded.result, created_at = excluded.created_at, ttl_ms = excluded.ttl_ms
|
|
||||||
`,
|
|
||||||
[key, JSON.stringify(result), Date.now(), ttl_ms],
|
|
||||||
"cache"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
initDatabase,
|
initDatabase,
|
||||||
getDatabase,
|
getDatabase,
|
||||||
queryOne,
|
queryOne,
|
||||||
queryAll,
|
queryAll,
|
||||||
run,
|
run,
|
||||||
getCachedExtension,
|
closeDatabase
|
||||||
cacheExtension,
|
|
||||||
getExtensionTitle,
|
|
||||||
deleteExtension,
|
|
||||||
closeDatabase,
|
|
||||||
getCache,
|
|
||||||
setCache
|
|
||||||
};
|
};
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
import { FastifyInstance, FastifyReply } from 'fastify';
|
|
||||||
import { proxyRequest, processM3U8Content, streamToReadable } from './proxy.service';
|
|
||||||
import { ProxyRequest } from '../../types';
|
|
||||||
|
|
||||||
async function proxyRoutes(fastify: FastifyInstance) {
|
|
||||||
fastify.get('/proxy', async (req: ProxyRequest, reply: FastifyReply) => {
|
|
||||||
const { url, referer, origin, userAgent } = req.query;
|
|
||||||
|
|
||||||
if (!url) {
|
|
||||||
return reply.code(400).send({ error: "No URL provided" });
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const { response, contentType, isM3U8 } = await proxyRequest(url, {
|
|
||||||
referer,
|
|
||||||
origin,
|
|
||||||
userAgent
|
|
||||||
});
|
|
||||||
|
|
||||||
reply.header('Access-Control-Allow-Origin', '*');
|
|
||||||
reply.header('Access-Control-Allow-Methods', 'GET, OPTIONS');
|
|
||||||
|
|
||||||
if (contentType) {
|
|
||||||
reply.header('Content-Type', contentType);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isM3U8) {
|
|
||||||
const text = await response.text();
|
|
||||||
const baseUrl = new URL(response.url);
|
|
||||||
const processedContent = processM3U8Content(text, baseUrl, {
|
|
||||||
referer,
|
|
||||||
origin,
|
|
||||||
userAgent
|
|
||||||
});
|
|
||||||
|
|
||||||
return processedContent;
|
|
||||||
} else {
|
|
||||||
return reply.send(streamToReadable(response.body!));
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (err) {
|
|
||||||
fastify.log.error(err);
|
|
||||||
return reply.code(500).send({ error: "Internal Server Error" });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export default proxyRoutes;
|
|
||||||
62
src/shared/queries.js
Normal file
62
src/shared/queries.js
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
const { queryOne, run } = require('./database');
|
||||||
|
|
||||||
|
async function getCachedExtension(extName, id) {
|
||||||
|
return queryOne(
|
||||||
|
"SELECT metadata FROM extension WHERE ext_name = ? AND id = ?",
|
||||||
|
[extName, id]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cacheExtension(extName, id, title, metadata) {
|
||||||
|
return run(
|
||||||
|
`
|
||||||
|
INSERT INTO extension (ext_name, id, title, metadata, updated_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(ext_name, id)
|
||||||
|
DO UPDATE SET
|
||||||
|
title = excluded.title,
|
||||||
|
metadata = excluded.metadata,
|
||||||
|
updated_at = ?
|
||||||
|
`,
|
||||||
|
[extName, id, title, JSON.stringify(metadata), Date.now(), Date.now()]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getExtensionTitle(extName, id) {
|
||||||
|
const sql = "SELECT title FROM extension WHERE ext_name = ? AND id = ?";
|
||||||
|
const row = await queryOne(sql, [extName, id], 'anilist');
|
||||||
|
return row ? row.title : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteExtension(extName) {
|
||||||
|
return run(
|
||||||
|
"DELETE FROM extension WHERE ext_name = ?",
|
||||||
|
[extName]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getCache(key) {
|
||||||
|
return queryOne("SELECT result, created_at, ttl_ms FROM cache WHERE key = ?", [key], "cache");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setCache(key, result, ttl_ms) {
|
||||||
|
return run(
|
||||||
|
`
|
||||||
|
INSERT INTO cache (key, result, created_at, ttl_ms)
|
||||||
|
VALUES (?, ?, ?, ?)
|
||||||
|
ON CONFLICT(key)
|
||||||
|
DO UPDATE SET result = excluded.result, created_at = excluded.created_at, ttl_ms = excluded.ttl_ms
|
||||||
|
`,
|
||||||
|
[key, JSON.stringify(result), Date.now(), ttl_ms],
|
||||||
|
"cache"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
getCachedExtension,
|
||||||
|
cacheExtension,
|
||||||
|
getExtensionTitle,
|
||||||
|
deleteExtension,
|
||||||
|
getCache,
|
||||||
|
setCache
|
||||||
|
};
|
||||||
@@ -5,12 +5,12 @@ import * as path from 'path';
|
|||||||
async function viewsRoutes(fastify: FastifyInstance) {
|
async function viewsRoutes(fastify: FastifyInstance) {
|
||||||
|
|
||||||
fastify.get('/', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'index.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'anime', 'index.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/books', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/books', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'books.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'books', 'books.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -20,47 +20,47 @@ async function viewsRoutes(fastify: FastifyInstance) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/gallery', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/gallery', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'gallery.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'gallery', 'gallery.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/gallery/:extension/*', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/gallery/:extension/*', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'gallery-image.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'gallery', 'image.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/gallery/favorites/*', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/gallery/favorites/*', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'gallery-image.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'gallery', 'image.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/anime/:id', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/anime/:id', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'anime.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'anime', 'anime.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/anime/:extension/*', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/anime/:extension/*', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'anime.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'anime', 'anime.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/watch/:id/:episode', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/watch/:id/:episode', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'watch.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'anime', 'watch.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/book/:id', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/book/:id', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'book.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'books', 'book.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/book/:extension/*', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/book/:extension/*', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'book.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'books', 'book.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.get('/read/:provider/:chapter/*', (req: FastifyRequest, reply: FastifyReply) => {
|
fastify.get('/read/:provider/:chapter/*', (req: FastifyRequest, reply: FastifyReply) => {
|
||||||
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'read.html'));
|
const stream = fs.createReadStream(path.join(__dirname, '..', '..', 'views', 'books', 'read.html'));
|
||||||
reply.type('text/html').send(stream);
|
reply.type('text/html').send(stream);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -141,7 +141,7 @@
|
|||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script src="../src/scripts/updateNotifier.js"></script>
|
<script src="../../src/scripts/updateNotifier.js"></script>
|
||||||
|
|
||||||
<script src="/src/scripts/anime/anime.js"></script>
|
<script src="/src/scripts/anime/anime.js"></script>
|
||||||
</body>
|
</body>
|
||||||
@@ -117,7 +117,7 @@
|
|||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script src="../src/scripts/updateNotifier.js"></script>
|
<script src="../../src/scripts/updateNotifier.js"></script>
|
||||||
<script src="../src/scripts/anime/animes.js"></script>
|
<script src="../../src/scripts/anime/animes.js"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@@ -101,7 +101,7 @@
|
|||||||
</main>
|
</main>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script src="../src/scripts/anime/player.js"></script>
|
<script src="../../src/scripts/anime/player.js"></script>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
document.addEventListener('DOMContentLoaded', () => {
|
document.addEventListener('DOMContentLoaded', () => {
|
||||||
@@ -181,6 +181,6 @@
|
|||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script src="../src/scripts/updateNotifier.js"></script>
|
<script src="../../src/scripts/updateNotifier.js"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@@ -124,7 +124,7 @@
|
|||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script src="../src/scripts/updateNotifier.js"></script>
|
<script src="../../src/scripts/updateNotifier.js"></script>
|
||||||
<script src="../src/scripts/books/book.js"></script>
|
<script src="../../src/scripts/books/book.js"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@@ -5,7 +5,7 @@
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>WaifuBoard Books</title>
|
<title>WaifuBoard Books</title>
|
||||||
<link rel="stylesheet" href="/views/css/books/books.css">
|
<link rel="stylesheet" href="/views/css/books/books.css">
|
||||||
<script src="../src/scripts/books/books.js" defer></script>
|
<script src="../../src/scripts/books/books.js" defer></script>
|
||||||
<link rel="stylesheet" href="/views/css/updateNotifier.css">
|
<link rel="stylesheet" href="/views/css/updateNotifier.css">
|
||||||
<link rel="icon" href="/public/assets/waifuboards.ico" type="image/x-icon">
|
<link rel="icon" href="/public/assets/waifuboards.ico" type="image/x-icon">
|
||||||
</head>
|
</head>
|
||||||
@@ -94,6 +94,6 @@
|
|||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script src="../src/scripts/updateNotifier.js"></script>
|
<script src="../../src/scripts/updateNotifier.js"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@@ -80,7 +80,7 @@
|
|||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script src="../src/scripts/updateNotifier.js"></script>
|
<script src="../../src/scripts/updateNotifier.js"></script>
|
||||||
<script src="../src/scripts/gallery/gallery.js"></script>
|
<script src="../../src/scripts/gallery/gallery.js"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
Reference in New Issue
Block a user