From d6a99bfeb4d95c6ce8189d7df8f156895ff0fd60 Mon Sep 17 00:00:00 2001 From: lenafx Date: Fri, 19 Dec 2025 19:23:29 +0100 Subject: [PATCH] animes & books page data is fetched auto now --- desktop/package-lock.json | 10 +++ desktop/package.json | 1 + desktop/server.js | 31 ++++++- desktop/src/api/anime/anime.service.ts | 117 +++++++++++-------------- desktop/src/api/books/books.service.ts | 65 ++++++-------- docker/package-lock.json | 10 +++ docker/package.json | 1 + docker/server.js | 36 ++++++-- docker/src/api/anime/anime.service.ts | 117 +++++++++++-------------- docker/src/api/books/books.service.ts | 65 ++++++-------- 10 files changed, 231 insertions(+), 222 deletions(-) diff --git a/desktop/package-lock.json b/desktop/package-lock.json index 3177433..fb228f8 100644 --- a/desktop/package-lock.json +++ b/desktop/package-lock.json @@ -19,6 +19,7 @@ "fastify": "^5.6.2", "jsonwebtoken": "^9.0.3", "node-addon-api": "^8.5.0", + "node-cron": "^4.2.1", "playwright-chromium": "^1.57.0", "sqlite3": "^5.1.7" }, @@ -5446,6 +5447,15 @@ "semver": "^7.3.5" } }, + "node_modules/node-cron": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-4.2.1.tgz", + "integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==", + "license": "ISC", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/node-gyp": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.1.0.tgz", diff --git a/desktop/package.json b/desktop/package.json index 95f01a5..e388a69 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -22,6 +22,7 @@ "fastify": "^5.6.2", "jsonwebtoken": "^9.0.3", "node-addon-api": "^8.5.0", + "node-cron": "^4.2.1", "playwright-chromium": "^1.57.0", "sqlite3": "^5.1.7" }, diff --git a/desktop/server.js b/desktop/server.js index faa374c..9f54151 100644 --- a/desktop/server.js +++ b/desktop/server.js @@ -4,14 +4,16 @@ const fastify = require("fastify")({ const path = require("path"); const jwt = require("jsonwebtoken"); +const cron = require("node-cron"); const { initHeadless } = require("./electron/shared/headless"); const { initDatabase } = require("./electron/shared/database"); const { loadExtensions } = require("./electron/shared/extensions"); const { init } = require("./electron/api/rpc/rpc.controller"); +const {refreshTrendingAnime, refreshTopAiringAnime} = require("./electron/api/anime/anime.service"); +const {refreshPopularBooks, refreshTrendingBooks} = require("./electron/api/books/books.service"); + const dotenv = require("dotenv"); - const isPackaged = process.env.IS_PACKAGED === "true"; - const envPath = isPackaged ? path.join(process.resourcesPath, ".env") : path.join(__dirname, ".env"); @@ -102,6 +104,8 @@ fastify.register(userRoutes, { prefix: "/api" }); fastify.register(anilistRoute, { prefix: "/api" }); fastify.register(listRoutes, { prefix: "/api" }); +const sleep = ms => new Promise(r => setTimeout(r, ms)); + const start = async () => { try { initDatabase("anilist"); @@ -110,12 +114,31 @@ const start = async () => { initDatabase("userdata"); init(); + const refreshAll = async () => { + await refreshTrendingAnime(); + await sleep(300); + await refreshTopAiringAnime(); + await sleep(300); + await refreshTrendingBooks(); + await sleep(300); + await refreshPopularBooks(); + }; + + cron.schedule("*/30 * * * *", async () => { + try { + await refreshAll(); + console.log("cache refreshed"); + } catch (e) { + console.error("refresh failed", e); + } + }); + await loadExtensions(); + await initHeadless(); + await refreshAll(); await fastify.listen({ port: 54322, host: "0.0.0.0" }); console.log(`Server running at http://localhost:54322`); - - await initHeadless(); } catch (err) { fastify.log.error(err); process.exit(1); diff --git a/desktop/src/api/anime/anime.service.ts b/desktop/src/api/anime/anime.service.ts index a7216f5..3e494bf 100644 --- a/desktop/src/api/anime/anime.service.ts +++ b/desktop/src/api/anime/anime.service.ts @@ -3,7 +3,6 @@ import { queryAll, queryOne } from '../../shared/database'; import {Anime, Episode, Extension, StreamData} from '../types'; const CACHE_TTL_MS = 24 * 60 * 60 * 1000; -const TTL = 60 * 60 * 6; const ANILIST_URL = "https://graphql.anilist.co"; @@ -79,6 +78,54 @@ const MEDIA_FIELDS = ` } `; +export async function refreshTrendingAnime(): Promise { + const query = ` + query { + Page(page: 1, perPage: 10) { + media(type: ANIME, sort: TRENDING_DESC) { ${MEDIA_FIELDS} } + } + } + `; + + const data = await fetchAniList(query, {}); + const list = data?.Page?.media || []; + const now = Math.floor(Date.now() / 1000); + + await queryOne("DELETE FROM trending"); + + let rank = 1; + for (const anime of list) { + await queryOne( + "INSERT INTO trending (rank, id, full_data, updated_at) VALUES (?, ?, ?, ?)", + [rank++, anime.id, JSON.stringify(anime), now] + ); + } +} + +export async function refreshTopAiringAnime(): Promise { + const query = ` + query { + Page(page: 1, perPage: 10) { + media(type: ANIME, status: RELEASING, sort: SCORE_DESC) { ${MEDIA_FIELDS} } + } + } + `; + + const data = await fetchAniList(query, {}); + const list = data?.Page?.media || []; + const now = Math.floor(Date.now() / 1000); + + await queryOne("DELETE FROM top_airing"); + + let rank = 1; + for (const anime of list) { + await queryOne( + "INSERT INTO top_airing (rank, id, full_data, updated_at) VALUES (?, ?, ?, ?)", + [rank++, anime.id, JSON.stringify(anime), now] + ); + } +} + async function fetchAniList(query: string, variables: any) { const res = await fetch(ANILIST_URL, { method: "POST", @@ -119,76 +166,16 @@ export async function getAnimeById(id: string | number): Promise { const rows = await queryAll( - "SELECT full_data, updated_at FROM trending ORDER BY rank ASC LIMIT 10" + "SELECT full_data FROM trending ORDER BY rank ASC LIMIT 10" ); - - if (rows.length) { - const expired = (Date.now() / 1000 - rows[0].updated_at) > TTL; - if (!expired) { - return rows.map((r: { full_data: string }) => JSON.parse(r.full_data)); - } - } - - const query = ` - query { - Page(page: 1, perPage: 10) { - media(type: ANIME, sort: TRENDING_DESC) { ${MEDIA_FIELDS} } - } - } - `; - - const data = await fetchAniList(query, {}); - const list = data?.Page?.media || []; - const now = Math.floor(Date.now() / 1000); - - await queryOne("DELETE FROM trending"); - let rank = 1; - - for (const anime of list) { - await queryOne( - "INSERT INTO trending (rank, id, full_data, updated_at) VALUES (?, ?, ?, ?)", - [rank++, anime.id, JSON.stringify(anime), now] - ); - } - - return list; + return rows.map((r: { full_data: string; }) => JSON.parse(r.full_data)); } export async function getTopAiringAnime(): Promise { const rows = await queryAll( - "SELECT full_data, updated_at FROM top_airing ORDER BY rank ASC LIMIT 10" + "SELECT full_data FROM top_airing ORDER BY rank ASC LIMIT 10" ); - - if (rows.length) { - const expired = (Date.now() / 1000 - rows[0].updated_at) > TTL; - if (!expired) { - return rows.map((r: { full_data: string }) => JSON.parse(r.full_data)); - } - } - - const query = ` - query { - Page(page: 1, perPage: 10) { - media(type: ANIME, status: RELEASING, sort: SCORE_DESC) { ${MEDIA_FIELDS} } - } - } - `; - - const data = await fetchAniList(query, {}); - const list = data?.Page?.media || []; - const now = Math.floor(Date.now() / 1000); - - await queryOne("DELETE FROM top_airing"); - let rank = 1; - - for (const anime of list) { - await queryOne( - "INSERT INTO top_airing (rank, id, full_data, updated_at) VALUES (?, ?, ?, ?)", - [rank++, anime.id, JSON.stringify(anime), now] - ); - } - - return list; + return rows.map((r: { full_data: string; }) => JSON.parse(r.full_data)); } export async function searchAnimeLocal(query: string): Promise { diff --git a/desktop/src/api/books/books.service.ts b/desktop/src/api/books/books.service.ts index 6869cf8..6b3e611 100644 --- a/desktop/src/api/books/books.service.ts +++ b/desktop/src/api/books/books.service.ts @@ -4,7 +4,6 @@ import { getAllExtensions, getBookExtensionsMap } from '../../shared/extensions' import { Book, Extension, ChapterWithProvider, ChapterContent } from '../types'; const CACHE_TTL_MS = 24 * 60 * 60 * 1000; -const TTL = 60 * 60 * 6; const ANILIST_URL = "https://graphql.anilist.co"; async function fetchAniList(query: string, variables: any) { @@ -134,25 +133,14 @@ export async function getBookById(id: string | number): Promise { - const rows = await queryAll( - "SELECT full_data, updated_at FROM trending_books ORDER BY rank ASC LIMIT 10" - ); - - if (rows.length) { - const expired = (Date.now() / 1000 - rows[0].updated_at) > TTL; - if (!expired) { - return rows.map((r: { full_data: string }) => JSON.parse(r.full_data)); - } - } - +export async function refreshTrendingBooks(): Promise { const query = ` - query { - Page(page: 1, perPage: 10) { - media(type: MANGA, sort: TRENDING_DESC) { ${MEDIA_FIELDS} } + query { + Page(page: 1, perPage: 10) { + media(type: MANGA, sort: TRENDING_DESC) { ${MEDIA_FIELDS} } + } } - } - `; + `; const data = await fetchAniList(query, {}); const list = data?.Page?.media || []; @@ -167,30 +155,16 @@ export async function getTrendingBooks(): Promise { [rank++, book.id, JSON.stringify(book), now] ); } - - return list; } - -export async function getPopularBooks(): Promise { - const rows = await queryAll( - "SELECT full_data, updated_at FROM popular_books ORDER BY rank ASC LIMIT 10" - ); - - if (rows.length) { - const expired = (Date.now() / 1000 - rows[0].updated_at) > TTL; - if (!expired) { - return rows.map((r: { full_data: string }) => JSON.parse(r.full_data)); - } - } - +export async function refreshPopularBooks(): Promise { const query = ` - query { - Page(page: 1, perPage: 10) { - media(type: MANGA, sort: POPULARITY_DESC) { ${MEDIA_FIELDS} } + query { + Page(page: 1, perPage: 10) { + media(type: MANGA, sort: POPULARITY_DESC) { ${MEDIA_FIELDS} } + } } - } - `; + `; const data = await fetchAniList(query, {}); const list = data?.Page?.media || []; @@ -205,10 +179,21 @@ export async function getPopularBooks(): Promise { [rank++, book.id, JSON.stringify(book), now] ); } - - return list; } +export async function getTrendingBooks(): Promise { + const rows = await queryAll( + "SELECT full_data FROM trending_books ORDER BY rank ASC LIMIT 10" + ); + return rows.map((r: { full_data: string; }) => JSON.parse(r.full_data)); +} + +export async function getPopularBooks(): Promise { + const rows = await queryAll( + "SELECT full_data FROM popular_books ORDER BY rank ASC LIMIT 10" + ); + return rows.map((r: { full_data: string; }) => JSON.parse(r.full_data)); +} export async function searchBooksLocal(query: string): Promise { if (!query || query.length < 2) { diff --git a/docker/package-lock.json b/docker/package-lock.json index 46aab53..23de5fb 100644 --- a/docker/package-lock.json +++ b/docker/package-lock.json @@ -17,6 +17,7 @@ "fastify": "^5.6.2", "jsonwebtoken": "^9.0.3", "node-addon-api": "^8.5.0", + "node-cron": "^4.2.1", "playwright-chromium": "^1.57.0", "sqlite3": "^5.1.7" }, @@ -2230,6 +2231,15 @@ "node": "^18 || ^20 || >= 21" } }, + "node_modules/node-cron": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-4.2.1.tgz", + "integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==", + "license": "ISC", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/node-gyp": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.1.0.tgz", diff --git a/docker/package.json b/docker/package.json index d89160c..ebd7bff 100644 --- a/docker/package.json +++ b/docker/package.json @@ -20,6 +20,7 @@ "fastify": "^5.6.2", "jsonwebtoken": "^9.0.3", "node-addon-api": "^8.5.0", + "node-cron": "^4.2.1", "playwright-chromium": "^1.57.0", "sqlite3": "^5.1.7" }, diff --git a/docker/server.js b/docker/server.js index 47bc1ca..f6b36c0 100644 --- a/docker/server.js +++ b/docker/server.js @@ -4,16 +4,15 @@ const fastify = require("fastify")({ const path = require("path"); const jwt = require("jsonwebtoken"); +const cron = require("node-cron"); const { initHeadless } = require("./electron/shared/headless"); const { initDatabase } = require("./electron/shared/database"); const { loadExtensions } = require("./electron/shared/extensions"); +const {refreshTrendingAnime, refreshTopAiringAnime} = require("./electron/api/anime/anime.service"); +const {refreshPopularBooks, refreshTrendingBooks} = require("./electron/api/books/books.service"); const dotenv = require("dotenv"); -const envPath = process.resourcesPath - ? path.join(process.resourcesPath, ".env") - : path.join(__dirname, ".env"); -// Attempt to load it and log the result to be sure -dotenv.config({ path: envPath }); +dotenv.config(); const viewsRoutes = require("./electron/views/views.routes"); const animeRoutes = require("./electron/api/anime/anime.routes"); @@ -65,6 +64,8 @@ fastify.register(userRoutes, { prefix: "/api" }); fastify.register(anilistRoute, { prefix: "/api" }); fastify.register(listRoutes, { prefix: "/api" }); +const sleep = ms => new Promise(r => setTimeout(r, ms)); + const start = async () => { try { initDatabase("anilist"); @@ -72,12 +73,31 @@ const start = async () => { initDatabase("cache"); initDatabase("userdata"); + const refreshAll = async () => { + await refreshTrendingAnime(); + await sleep(300); + await refreshTopAiringAnime(); + await sleep(300); + await refreshTrendingBooks(); + await sleep(300); + await refreshPopularBooks(); + }; + + cron.schedule("*/30 * * * *", async () => { + try { + await refreshAll(); + console.log("cache refreshed"); + } catch (e) { + console.error("refresh failed", e); + } + }); + await loadExtensions(); + await initHeadless(); + await refreshAll(); await fastify.listen({ port: 54322, host: "0.0.0.0" }); - console.log(`Server is now running!`); - - await initHeadless(); + console.log(`Server running at http://localhost:54322`); } catch (err) { fastify.log.error(err); process.exit(1); diff --git a/docker/src/api/anime/anime.service.ts b/docker/src/api/anime/anime.service.ts index a7216f5..3e494bf 100644 --- a/docker/src/api/anime/anime.service.ts +++ b/docker/src/api/anime/anime.service.ts @@ -3,7 +3,6 @@ import { queryAll, queryOne } from '../../shared/database'; import {Anime, Episode, Extension, StreamData} from '../types'; const CACHE_TTL_MS = 24 * 60 * 60 * 1000; -const TTL = 60 * 60 * 6; const ANILIST_URL = "https://graphql.anilist.co"; @@ -79,6 +78,54 @@ const MEDIA_FIELDS = ` } `; +export async function refreshTrendingAnime(): Promise { + const query = ` + query { + Page(page: 1, perPage: 10) { + media(type: ANIME, sort: TRENDING_DESC) { ${MEDIA_FIELDS} } + } + } + `; + + const data = await fetchAniList(query, {}); + const list = data?.Page?.media || []; + const now = Math.floor(Date.now() / 1000); + + await queryOne("DELETE FROM trending"); + + let rank = 1; + for (const anime of list) { + await queryOne( + "INSERT INTO trending (rank, id, full_data, updated_at) VALUES (?, ?, ?, ?)", + [rank++, anime.id, JSON.stringify(anime), now] + ); + } +} + +export async function refreshTopAiringAnime(): Promise { + const query = ` + query { + Page(page: 1, perPage: 10) { + media(type: ANIME, status: RELEASING, sort: SCORE_DESC) { ${MEDIA_FIELDS} } + } + } + `; + + const data = await fetchAniList(query, {}); + const list = data?.Page?.media || []; + const now = Math.floor(Date.now() / 1000); + + await queryOne("DELETE FROM top_airing"); + + let rank = 1; + for (const anime of list) { + await queryOne( + "INSERT INTO top_airing (rank, id, full_data, updated_at) VALUES (?, ?, ?, ?)", + [rank++, anime.id, JSON.stringify(anime), now] + ); + } +} + async function fetchAniList(query: string, variables: any) { const res = await fetch(ANILIST_URL, { method: "POST", @@ -119,76 +166,16 @@ export async function getAnimeById(id: string | number): Promise { const rows = await queryAll( - "SELECT full_data, updated_at FROM trending ORDER BY rank ASC LIMIT 10" + "SELECT full_data FROM trending ORDER BY rank ASC LIMIT 10" ); - - if (rows.length) { - const expired = (Date.now() / 1000 - rows[0].updated_at) > TTL; - if (!expired) { - return rows.map((r: { full_data: string }) => JSON.parse(r.full_data)); - } - } - - const query = ` - query { - Page(page: 1, perPage: 10) { - media(type: ANIME, sort: TRENDING_DESC) { ${MEDIA_FIELDS} } - } - } - `; - - const data = await fetchAniList(query, {}); - const list = data?.Page?.media || []; - const now = Math.floor(Date.now() / 1000); - - await queryOne("DELETE FROM trending"); - let rank = 1; - - for (const anime of list) { - await queryOne( - "INSERT INTO trending (rank, id, full_data, updated_at) VALUES (?, ?, ?, ?)", - [rank++, anime.id, JSON.stringify(anime), now] - ); - } - - return list; + return rows.map((r: { full_data: string; }) => JSON.parse(r.full_data)); } export async function getTopAiringAnime(): Promise { const rows = await queryAll( - "SELECT full_data, updated_at FROM top_airing ORDER BY rank ASC LIMIT 10" + "SELECT full_data FROM top_airing ORDER BY rank ASC LIMIT 10" ); - - if (rows.length) { - const expired = (Date.now() / 1000 - rows[0].updated_at) > TTL; - if (!expired) { - return rows.map((r: { full_data: string }) => JSON.parse(r.full_data)); - } - } - - const query = ` - query { - Page(page: 1, perPage: 10) { - media(type: ANIME, status: RELEASING, sort: SCORE_DESC) { ${MEDIA_FIELDS} } - } - } - `; - - const data = await fetchAniList(query, {}); - const list = data?.Page?.media || []; - const now = Math.floor(Date.now() / 1000); - - await queryOne("DELETE FROM top_airing"); - let rank = 1; - - for (const anime of list) { - await queryOne( - "INSERT INTO top_airing (rank, id, full_data, updated_at) VALUES (?, ?, ?, ?)", - [rank++, anime.id, JSON.stringify(anime), now] - ); - } - - return list; + return rows.map((r: { full_data: string; }) => JSON.parse(r.full_data)); } export async function searchAnimeLocal(query: string): Promise { diff --git a/docker/src/api/books/books.service.ts b/docker/src/api/books/books.service.ts index 6869cf8..6b3e611 100644 --- a/docker/src/api/books/books.service.ts +++ b/docker/src/api/books/books.service.ts @@ -4,7 +4,6 @@ import { getAllExtensions, getBookExtensionsMap } from '../../shared/extensions' import { Book, Extension, ChapterWithProvider, ChapterContent } from '../types'; const CACHE_TTL_MS = 24 * 60 * 60 * 1000; -const TTL = 60 * 60 * 6; const ANILIST_URL = "https://graphql.anilist.co"; async function fetchAniList(query: string, variables: any) { @@ -134,25 +133,14 @@ export async function getBookById(id: string | number): Promise { - const rows = await queryAll( - "SELECT full_data, updated_at FROM trending_books ORDER BY rank ASC LIMIT 10" - ); - - if (rows.length) { - const expired = (Date.now() / 1000 - rows[0].updated_at) > TTL; - if (!expired) { - return rows.map((r: { full_data: string }) => JSON.parse(r.full_data)); - } - } - +export async function refreshTrendingBooks(): Promise { const query = ` - query { - Page(page: 1, perPage: 10) { - media(type: MANGA, sort: TRENDING_DESC) { ${MEDIA_FIELDS} } + query { + Page(page: 1, perPage: 10) { + media(type: MANGA, sort: TRENDING_DESC) { ${MEDIA_FIELDS} } + } } - } - `; + `; const data = await fetchAniList(query, {}); const list = data?.Page?.media || []; @@ -167,30 +155,16 @@ export async function getTrendingBooks(): Promise { [rank++, book.id, JSON.stringify(book), now] ); } - - return list; } - -export async function getPopularBooks(): Promise { - const rows = await queryAll( - "SELECT full_data, updated_at FROM popular_books ORDER BY rank ASC LIMIT 10" - ); - - if (rows.length) { - const expired = (Date.now() / 1000 - rows[0].updated_at) > TTL; - if (!expired) { - return rows.map((r: { full_data: string }) => JSON.parse(r.full_data)); - } - } - +export async function refreshPopularBooks(): Promise { const query = ` - query { - Page(page: 1, perPage: 10) { - media(type: MANGA, sort: POPULARITY_DESC) { ${MEDIA_FIELDS} } + query { + Page(page: 1, perPage: 10) { + media(type: MANGA, sort: POPULARITY_DESC) { ${MEDIA_FIELDS} } + } } - } - `; + `; const data = await fetchAniList(query, {}); const list = data?.Page?.media || []; @@ -205,10 +179,21 @@ export async function getPopularBooks(): Promise { [rank++, book.id, JSON.stringify(book), now] ); } - - return list; } +export async function getTrendingBooks(): Promise { + const rows = await queryAll( + "SELECT full_data FROM trending_books ORDER BY rank ASC LIMIT 10" + ); + return rows.map((r: { full_data: string; }) => JSON.parse(r.full_data)); +} + +export async function getPopularBooks(): Promise { + const rows = await queryAll( + "SELECT full_data FROM popular_books ORDER BY rank ASC LIMIT 10" + ); + return rows.map((r: { full_data: string; }) => JSON.parse(r.full_data)); +} export async function searchBooksLocal(query: string): Promise { if (!query || query.length < 2) {