ItsSkaiya/WaifuBoard
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 8 Wiki Activity

support for multiple users

Browse Source
This commit is contained in:
lenafx
2025-12-06 01:21:19 +01:00
parent 4e2875579c
commit 2df7625657
20 changed files with 2313 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
src/shared/database.js
View File

@@ -8,9 +8,70 @@ const databases = new Map();
const DEFAULT_PATHS = {
anilist: path.join(process.cwd(), 'src', 'metadata', 'anilist_anime.db'),
favorites: path.join(os.homedir(), "WaifuBoards", "favorites.db"),
cache: path.join(os.homedir(), "WaifuBoards", "cache.db")
cache: path.join(os.homedir(), "WaifuBoards", "cache.db"),
userdata: path.join(os.homedir(), "WaifuBoards", "user_data.db")
};
async function ensureUserDataDB(dbPath) {
const dir = path.dirname(dbPath);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
const db = new sqlite3.Database(
dbPath,
sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE
);
return new Promise((resolve, reject) => {
const schema = `
-- Tabla 1: User
CREATE TABLE IF NOT EXISTS User (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username TEXT NOT NULL UNIQUE,
profile_picture_url TEXT,
email TEXT UNIQUE,
password_hash TEXT,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- Tabla 2: UserIntegration (✅ ACTUALIZADA)
CREATE TABLE IF NOT EXISTS UserIntegration (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL UNIQUE,
platform TEXT NOT NULL DEFAULT 'AniList',
access_token TEXT NOT NULL,
refresh_token TEXT NOT NULL,
token_type TEXT NOT NULL,
anilist_user_id INTEGER NOT NULL,
expires_at DATETIME NOT NULL,
FOREIGN KEY (user_id) REFERENCES User(id) ON DELETE CASCADE
);
-- Tabla 3: ListEntry
CREATE TABLE IF NOT EXISTS ListEntry (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
anime_id INTEGER NOT NULL,
external_id INTEGER,
source TEXT NOT NULL,
status TEXT NOT NULL,
progress INTEGER NOT NULL DEFAULT 0,
score INTEGER,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (user_id, anime_id),
FOREIGN KEY (user_id) REFERENCES User(id) ON DELETE CASCADE
);
`;
db.exec(schema, (err) => {
if (err) reject(err);
else resolve(true);
});
});
}
async function ensureExtensionsTable(db) {
return new Promise((resolve, reject) => {
db.exec(`
@@ -128,6 +189,11 @@ function initDatabase(name = 'anilist', dbPath = null, readOnly = false) {
}
}
if (name === "userdata") {
ensureUserDataDB(finalPath)
.catch(err => console.error("Error creando userdata:", err));
}
const mode = readOnly ? sqlite3.OPEN_READONLY : (sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE);
const db = new sqlite3.Database(finalPath, mode, (err) => {

167
src/shared/headless.js
View File

@@ -10,8 +10,6 @@ const BLOCK_LIST = [
"map", "cdn.ampproject.org", "googletagmanager"
];
const ALLOWED_SCRIPTS = [];
async function initHeadless() {
if (browser && browser.isConnected()) return;
@@ -30,7 +28,6 @@ async function initHeadless() {
"--mute-audio",
"--no-first-run",
"--no-zygote",
"--single-process",
"--disable-software-rasterizer",
"--disable-client-side-phishing-detection",
"--no-default-browser-check",
@@ -43,42 +40,103 @@ async function initHeadless() {
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36"
});
} catch (error) {
console.error("Error al inicializar browser:", error);
console.error("Error initializing browser:", error);
throw error;
}
}
async function safeEvaluate(page, fn, ...args) {
const maxAttempts = 3;
for (let i = 0; i < maxAttempts; i++) {
try {
// Checkeo de estado de página antes de intentar evaluar
if (page.isClosed()) {
throw new Error("Page is closed before evaluation");
}
return await Promise.race([
page.evaluate(fn, ...args),
new Promise((_, reject) =>
// Timeout más corto podría ser más seguro, e.g., 20000ms
setTimeout(() => reject(new Error("Evaluate timeout")), 30000)
)
]);
} catch (error) {
const errorMsg = (error.message || "").toLowerCase();
const isLastAttempt = i === maxAttempts - 1;
// Priorizar errores irrecuperables de contexto/página cerrada
if (
page.isClosed() ||
errorMsg.includes("closed") ||
errorMsg.includes("target closed") ||
errorMsg.includes("session closed")
) {
console.error("Page context lost or closed, throwing fatal error.");
throw error; // Lanzar inmediatamente, no tiene sentido reintentar
}
// Reintentar solo por errores transitorios de ejecución
if (!isLastAttempt && (
errorMsg.includes("execution context was destroyed") ||
errorMsg.includes("cannot find context") ||
errorMsg.includes("timeout")
)) {
console.warn(`Evaluate attempt ${i + 1} failed, retrying...`, error.message);
await new Promise(r => setTimeout(r, 500 * (i + 1)));
continue;
}
throw error;
}
}
}
async function turboScroll(page) {
try {
await page.evaluate(() => {
if (page.isClosed()) {
console.warn("Page closed, skipping scroll");
return;
}
await safeEvaluate(page, () => {
return new Promise((resolve) => {
let last = 0;
let same = 0;
const timer = setInterval(() => {
const h = document.body.scrollHeight;
window.scrollTo(0, h);
if (h === last) {
same++;
if (same >= 5) {
clearInterval(timer);
resolve();
let lastHeight = 0;
let sameCount = 0;
const scrollInterval = setInterval(() => {
try {
const currentHeight = document.body.scrollHeight;
window.scrollTo(0, currentHeight);
if (currentHeight === lastHeight) {
sameCount++;
if (sameCount >= 5) {
clearInterval(scrollInterval);
resolve();
}
} else {
sameCount = 0;
lastHeight = currentHeight;
}
} else {
same = 0;
last = h;
} catch (err) {
clearInterval(scrollInterval);
resolve();
}
}, 20);
// Safety timeout
setTimeout(() => {
clearInterval(timer);
clearInterval(scrollInterval);
resolve();
}, 10000);
});
});
} catch (error) {
console.error("Error en turboScroll:", error.message);
// No lanzamos el error, continuamos
console.error("Error in turboScroll:", error.message);
}
}
@@ -90,7 +148,6 @@ async function scrape(url, handler, options = {}) {
scrollToBottom = false,
renderWaitTime = 0,
loadImages = true,
blockScripts = true,
retries = 3,
retryDelay = 1000
} = options;
@@ -101,7 +158,7 @@ async function scrape(url, handler, options = {}) {
let page = null;
try {
// Verificar que el browser esté activo
if (!browser || !browser.isConnected()) {
await initHeadless();
}
@@ -109,7 +166,10 @@ async function scrape(url, handler, options = {}) {
page = await context.newPage();
const requests = [];
// Listener para requests
page.on("close", () => {
console.warn("Page closed unexpectedly");
});
page.on("request", req => {
requests.push({
url: req.url(),
@@ -118,7 +178,6 @@ async function scrape(url, handler, options = {}) {
});
});
// Route para bloquear recursos
await page.route("**/*", (route) => {
const req = route.request();
const resUrl = req.url().toLowerCase();
@@ -128,9 +187,7 @@ async function scrape(url, handler, options = {}) {
type === "font" ||
type === "stylesheet" ||
type === "media" ||
type === "manifest" ||
type === "other" ||
(blockScripts && type === "script" && !ALLOWED_SCRIPTS.some(k => resUrl.includes(k)))
type === "other"
) {
return route.abort("blockedbyclient");
}
@@ -148,66 +205,71 @@ async function scrape(url, handler, options = {}) {
route.continue();
});
// Navegar a la URL
await page.goto(url, { waitUntil, timeout });
// Esperar selector si se especifica
if (!page.isClosed()) {
await page.waitForTimeout(500);
}
if (waitSelector) {
try {
await page.waitForSelector(waitSelector, { timeout: Math.min(timeout, 5000) });
await page.waitForSelector(waitSelector, {
timeout: Math.min(timeout, 5000)
});
} catch (e) {
console.warn(`Selector '${waitSelector}' no encontrado, continuando...`);
console.warn(`Selector '${waitSelector}' not found, continuing...`);
}
}
// Scroll si es necesario
if (scrollToBottom) {
await turboScroll(page);
}
// Tiempo de espera adicional para renderizado
if (renderWaitTime > 0) {
await page.waitForTimeout(renderWaitTime);
}
// Ejecutar el handler personalizado
const result = await handler(page);
if (page.isClosed()) {
throw new Error("Page closed before handler execution");
}
const result = await handler(page, safeEvaluate);
// Cerrar la página antes de retornar
await page.close();
return { result, requests };
} catch (error) {
lastError = error;
console.error(`[Intento ${attempt}/${retries}] Error durante el scraping de ${url}:`, error.message);
console.error(`[Attempt ${attempt}/${retries}] Error scraping ${url}:`, error.message);
// Cerrar página si está abierta
if (page && !page.isClosed()) {
try {
await page.close();
} catch (closeError) {
console.error("Error al cerrar página:", closeError.message);
console.error("Error closing page:", closeError.message);
}
}
// Si el browser está cerrado, limpiar referencias
if (error.message.includes("closed") || error.message.includes("Target closed")) {
console.log("Browser cerrado detectado, reiniciando...");
if (
error.message.includes("closed") ||
error.message.includes("Target closed") ||
error.message.includes("Session closed")
) {
console.log("Browser closure detected, reinitializing...");
await closeScraper();
}
// Si no es el último intento, esperar antes de reintentar
if (attempt < retries) {
const delay = retryDelay * attempt; // Backoff exponencial
console.log(`Reintentando en ${delay}ms...`);
const delay = retryDelay * attempt;
console.log(`Retrying in ${delay}ms...`);
await new Promise(r => setTimeout(r, delay));
}
}
}
// Si llegamos aquí, todos los intentos fallaron
console.error(`Todos los intentos fallaron para ${url}`);
console.error(`All attempts failed for ${url}`);
throw lastError || new Error("Scraping failed after all retries");
}
@@ -218,7 +280,7 @@ async function closeScraper() {
context = null;
}
} catch (error) {
console.error("Error cerrando context:", error.message);
console.error("Error closing context:", error.message);
}
try {
@@ -227,12 +289,13 @@ async function closeScraper() {
browser = null;
}
} catch (error) {
console.error("Error cerrando browser:", error.message);
console.error("Error closing browser:", error.message);
}
}
module.exports = {
initHeadless,
scrape,
closeScraper
closeScraper,
safeEvaluate
};