8 Commits

Author SHA1 Message Date
7490f269b0 Updated version in updateNotifier 2025-12-17 16:37:56 -05:00
c7f919fe18 fixed an error where app wouldnt launch 2025-12-17 21:32:34 +01:00
7c85d91b85 Merge branch 'main' of https://git.waifuboard.app/ItsSkaiya/WaifuBoard 2025-12-17 13:00:35 -05:00
c11a1eed35 Docker server has responsiveness now 2025-12-17 13:00:15 -05:00
7544f56ba9 readme & fix on reader 2025-12-17 16:43:26 +01:00
1a01d29f19 Removed docker stuff from the desktop files. 2025-12-16 21:54:53 -05:00
315c2e911b Added back the README.md to the desktop version 2025-12-16 21:52:45 -05:00
28ff6ccc68 Organized the differences between server and docker versions.
We are launching a docker version (server version) today so we want to just organize the repo
so its easier to navigate.
2025-12-16 21:50:22 -05:00
198 changed files with 25008 additions and 51 deletions

16
.gitignore vendored
View File

@@ -1,5 +1,11 @@
node_modules desktop/node_modules
electron desktop/electron
dist desktop/dist
.env desktop/.env
build desktop/build
docker/node_modules
docker/electron
docker/dist
docker/.env
docker/build

View File

@@ -1,31 +1,55 @@
# 🎀 WaifuBoard # 🎀 WaifuBoard
**Lightweight all-in-one app for boorus, manga and light novels — no sources included, total freedom via extensions.** **Lightweight all-in-one app for boorus, anime, manga and light novels — no sources included, total freedom via extensions.**
<img src="public/banner.png" alt="WaifuBoard Hero" width="100%"/> <img src="assets/hero.png" alt="WaifuBoard Hero" width="100%"/>
<div align="center"> <div align="center">
[![Windows](https://img.shields.io/badge/Windows-SUPPORTED-0078D6?style=for-the-badge&logo=windows&logoColor=white)](https://waifuboard.app) [![Windows](https://img.shields.io/badge/Windows-SUPPORTED-0078D6?style=for-the-badge&logo=windows&logoColor=white)](https://waifuboard.app)
[![Latest](https://img.shields.io/gitea/v/release/ItsSkaiya/WaifuBoard?gitea_url=https://git.waifuboard.app&style=for-the-badge)](https://git.waifuboard.app/ItsSkaiya/WaifuBoard/releases/latest) [![Latest](https://img.shields.io/gitea/v/release/ItsSkaiya/WaifuBoard?gitea_url=https://git.waifuboard.app&style=for-the-badge)](https://git.waifuboard.app/ItsSkaiya/WaifuBoard/releases/latest)
[![Extensions](https://img.shields.io/badge/Extensions-Repository-8257e5?style=for-the-badge)](https://git.waifuboard.app/ItsSkaiya/WaifuBoard-Extensions) [![Extensions](https://img.shields.io/badge/Extensions-Repository-8257e5?style=for-the-badge)](https://git.waifuboard.app/ItsSkaiya/WaifuBoard-Extensions)
**[Website](https://waifuboard.app)** • **[Documentation](https://waifuboard.app/docs)** • **[Download Latest](https://git.waifuboard.app/ItsSkaiya/WaifuBoard/releases/latest)** • **[Discord (soon)](#)** **[Website](https://waifuboard.app)** • **[Documentation](https://waifuboard.app/docs)** • **[Download Latest](https://git.waifuboard.app/ItsSkaiya/WaifuBoard/releases/latest)** • **[Discord](https://discord.gg/DAVNf7erRJ)**
</div> </div>
## 🚀 What is WaifuBoard?
**A lightweight and privacy-friendly desktop app designed around modular extensions.** WaifuBoard delivers a clean, organized reading experience for boorus, manga, and light novels, powered entirely by community-made sources.
## ✨ Features ## ✨ Features
- Lightweight - Lightweight
- Discord Rich Presence - Discord Rich Presence
- Super clean & fast UI - Local lists & anilist integration
- Built-in **Extension Marketplace** - Multi user support
- Server version (Coming soon!)
- Fully open-source & community-driven - Fully open-source & community-driven
- Future mobile ports planned
> ⚠️ **Note**
>
> Release candidates may contain breaking changes and unresolved bugs.
> Expect frequent updates and major changes between versions.
## 🖼️ Gallery
### 👥 Multi-User Profiles
<img src="assets/users.gif" alt="Multi-user profiles" width="100%"/>
### 📖 Reader
#### Manga
<img src="assets/manga.png" alt="Manga reader" width="100%"/>
#### Manhwa
<img src="assets/manhwa.png" alt="Manhwa reader" width="100%"/>
#### Light Novel
<img src="assets/novel.gif" alt="Light novel reader" width="100%"/>
### 📅 Schedule & Tracking
<img src="assets/schedule.png" alt="Anime schedule" width="100%"/>
### 📚 Lists (Local & AniList)
<img src="assets/list.gif" alt="Lists and tracking" width="100%"/>
## 🖥️ Download & Platform Support ## 🖥️ Download & Platform Support

BIN
assets/hero.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.8 MiB

BIN
assets/list.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 MiB

BIN
assets/manga.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 756 KiB

BIN
assets/manhwa.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 577 KiB

BIN
assets/novel.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 MiB

BIN
assets/schedule.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 227 KiB

BIN
assets/users.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.2 MiB

5
desktop/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
node_modules
electron
dist
.env
build

121
desktop/main.js Normal file
View File

@@ -0,0 +1,121 @@
const { app, BrowserWindow, ipcMain } = require('electron');
const { fork } = require('child_process');
const path = require('path');
const log = require('electron-log');
log.transports.file.level = 'info';
log.format = '[{y}-{m}-{d} {h}:{i}:{s}.{ms}] [{level}] {text}';
let win;
let backend;
const net = require('net');
function waitForServer(port, host = '127.0.0.1', timeout = 10000) {
return new Promise((resolve, reject) => {
const start = Date.now();
const check = () => {
const socket = new net.Socket();
socket
.once('connect', () => {
socket.destroy();
resolve();
})
.once('error', () => {
socket.destroy();
if (Date.now() - start > timeout) {
reject(new Error('Backend timeout'));
} else {
setTimeout(check, 200);
}
})
.connect(port, host);
};
check();
});
}
function startBackend() {
backend = fork(path.join(__dirname, 'server.js'), [], {
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
env: {
...process.env,
IS_PACKAGED: app.isPackaged ? 'true' : 'false'
}
});
log.info('Starting backend process...');
backend.stdout.on('data', (data) => {
log.info(`[Backend]: ${data.toString().trim()}`);
});
backend.stderr.on('data', (data) => {
log.error(`[Backend ERROR]: ${data.toString().trim()}`);
});
backend.on('exit', (code) => {
log.warn(`Backend process exited with code: ${code}`);
});
}
function createWindow() {
win = new BrowserWindow({
width: 1200,
height: 800,
frame: false,
titleBarStyle: "hidden",
webPreferences: {
preload: path.join(__dirname, "preload.js"),
nodeIntegration: false,
contextIsolation: true
}
});
win.setMenu(null);
win.loadURL('http://localhost:54322');
win.on('closed', () => {
win = null;
});
}
ipcMain.on("win:minimize", () => win.minimize());
ipcMain.on("win:maximize", () => {
if (win.isMaximized()) {
win.unmaximize();
} else {
win.maximize();
}
});
ipcMain.on("win:close", () => win.close());
process.on('uncaughtException', (err) => {
log.error('Critical unhandled error in Main:', err);
});
app.whenReady().then(async () => {
log.info('--- Application Started ---');
console.log("Logs location:", log.transports.file.getFile().path);
startBackend();
await waitForServer(54322);
createWindow();
app.on('activate', () => {
if (BrowserWindow.getAllWindows().length === 0) createWindow();
});
});
app.on('window-all-closed', () => {
log.info('Closing all windows...');
if (backend) {
backend.kill();
log.info('Backend process terminated.');
}
if (process.platform !== 'darwin') {
app.quit();
}
});

View File

@@ -11,10 +11,11 @@
"dependencies": { "dependencies": {
"@fastify/static": "^8.3.0", "@fastify/static": "^8.3.0",
"@ryuziii/discord-rpc": "^1.0.1-rc.1", "@ryuziii/discord-rpc": "^1.0.1-rc.1",
"bcrypt": "^6.0.0", "bcryptjs": "^3.0.3",
"bindings": "^1.5.0", "bindings": "^1.5.0",
"cheerio": "^1.1.2", "cheerio": "^1.1.2",
"dotenv": "^17.2.3", "dotenv": "^17.2.3",
"electron-log": "^5.4.3",
"fastify": "^5.6.2", "fastify": "^5.6.2",
"jsonwebtoken": "^9.0.3", "jsonwebtoken": "^9.0.3",
"node-addon-api": "^8.5.0", "node-addon-api": "^8.5.0",
@@ -1972,18 +1973,13 @@
], ],
"license": "MIT" "license": "MIT"
}, },
"node_modules/bcrypt": { "node_modules/bcryptjs": {
"version": "6.0.0", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-6.0.0.tgz", "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-3.0.3.tgz",
"integrity": "sha512-cU8v/EGSrnH+HnxV2z0J7/blxH8gq7Xh2JFT6Aroax7UohdmiJJlxApMxtKfuI7z68NvvVcmR78k2LbT6efhRg==", "integrity": "sha512-GlF5wPWnSa/X5LKM1o0wz0suXIINz1iHRLvTS+sLyi7XPbe5ycmYI3DlZqVGZZtDgl4DmasFg7gOB3JYbphV5g==",
"hasInstallScript": true, "license": "BSD-3-Clause",
"license": "MIT", "bin": {
"dependencies": { "bcrypt": "bin/bcrypt"
"node-addon-api": "^8.3.0",
"node-gyp-build": "^4.8.4"
},
"engines": {
"node": ">= 18"
} }
}, },
"node_modules/bindings": { "node_modules/bindings": {
@@ -3402,6 +3398,15 @@
"node": ">= 10.0.0" "node": ">= 10.0.0"
} }
}, },
"node_modules/electron-log": {
"version": "5.4.3",
"resolved": "https://registry.npmjs.org/electron-log/-/electron-log-5.4.3.tgz",
"integrity": "sha512-sOUsM3LjZdugatazSQ/XTyNcw8dfvH1SYhXWiJyfYodAAKOZdHs0txPiLDXFzOZbhXgAgshQkshH2ccq0feyLQ==",
"license": "MIT",
"engines": {
"node": ">= 14"
}
},
"node_modules/electron-publish": { "node_modules/electron-publish": {
"version": "26.0.11", "version": "26.0.11",
"resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-26.0.11.tgz", "resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-26.0.11.tgz",
@@ -5365,17 +5370,6 @@
"node": "^20.17.0 || >=22.9.0" "node": "^20.17.0 || >=22.9.0"
} }
}, },
"node_modules/node-gyp-build": {
"version": "4.8.4",
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz",
"integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==",
"license": "MIT",
"bin": {
"node-gyp-build": "bin.js",
"node-gyp-build-optional": "optional.js",
"node-gyp-build-test": "build-test.js"
}
},
"node_modules/node-gyp/node_modules/chownr": { "node_modules/node-gyp/node_modules/chownr": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",

View File

@@ -4,10 +4,8 @@
"description": "", "description": "",
"main": "main.js", "main": "main.js",
"scripts": { "scripts": {
"build": "tsc", "start": "tsc && electron .",
"start": "tsc && node server.js", "dist": "tsc && electron-builder"
"electron": "tsc && electron .",
"dist": "npm run build && electron-builder"
}, },
"keywords": [], "keywords": [],
"author": "", "author": "",
@@ -16,10 +14,11 @@
"dependencies": { "dependencies": {
"@fastify/static": "^8.3.0", "@fastify/static": "^8.3.0",
"@ryuziii/discord-rpc": "^1.0.1-rc.1", "@ryuziii/discord-rpc": "^1.0.1-rc.1",
"bcrypt": "^6.0.0", "bcryptjs": "^3.0.3",
"bindings": "^1.5.0", "bindings": "^1.5.0",
"cheerio": "^1.1.2", "cheerio": "^1.1.2",
"dotenv": "^17.2.3", "dotenv": "^17.2.3",
"electron-log": "^5.4.3",
"fastify": "^5.6.2", "fastify": "^5.6.2",
"jsonwebtoken": "^9.0.3", "jsonwebtoken": "^9.0.3",
"node-addon-api": "^8.5.0", "node-addon-api": "^8.5.0",
@@ -50,7 +49,11 @@
"public/assets/*" "public/assets/*"
], ],
"extraResources": [ "extraResources": [
"./.env" {
"from": "C:\\Users\\synta\\AppData\\Local\\ms-playwright\\chromium_headless_shell-1200",
"to": "playwright/chromium"
},
".env"
], ],
"win": { "win": {
"target": "portable", "target": "portable",

View File

Before

Width:  |  Height:  |  Size: 2.0 KiB

After

Width:  |  Height:  |  Size: 2.0 KiB

View File

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

View File

Before

Width:  |  Height:  |  Size: 123 KiB

After

Width:  |  Height:  |  Size: 123 KiB

View File

@@ -9,13 +9,14 @@ const { initDatabase } = require("./electron/shared/database");
const { loadExtensions } = require("./electron/shared/extensions"); const { loadExtensions } = require("./electron/shared/extensions");
const { init } = require("./electron/api/rpc/rpc.controller"); const { init } = require("./electron/api/rpc/rpc.controller");
const dotenv = require("dotenv"); const dotenv = require("dotenv");
const envPath = process.resourcesPath
? path.join(process.resourcesPath, ".env")
: path.join(__dirname, ".env");
// Attempt to load it and log the result to be sure const isPackaged = process.env.IS_PACKAGED === "true";
dotenv.config({ path: envPath });
const envPath = isPackaged
? path.join(process.resourcesPath, ".env")
: path.join(__dirname, ".env");
dotenv.config({ path: envPath, override: false });
const viewsRoutes = require("./electron/views/views.routes"); const viewsRoutes = require("./electron/views/views.routes");
const animeRoutes = require("./electron/api/anime/anime.routes"); const animeRoutes = require("./electron/api/anime/anime.routes");
const booksRoutes = require("./electron/api/books/books.routes"); const booksRoutes = require("./electron/api/books/books.routes");

View File

@@ -0,0 +1,186 @@
import {queryAll, queryOne, run} from '../../shared/database';
import bcrypt from 'bcryptjs';
const USER_DB_NAME = 'userdata';
const SALT_ROUNDS = 10;
interface User {
id: number;
username: string;
profile_picture_url: string | null;
has_password: boolean;
}
export async function userExists(id: number): Promise<boolean> {
const sql = 'SELECT 1 FROM User WHERE id = ?';
const row = await queryOne(sql, [id], USER_DB_NAME);
return !!row;
}
export async function createUser(username: string, profilePictureUrl?: string, password?: string): Promise<{ lastID: number }> {
let passwordHash = null;
if (password && password.trim()) {
passwordHash = await bcrypt.hash(password.trim(), SALT_ROUNDS);
}
const sql = `
INSERT INTO User (username, profile_picture_url, password_hash)
VALUES (?, ?, ?)
`;
const params = [username, profilePictureUrl || null, passwordHash];
const result = await run(sql, params, USER_DB_NAME);
return { lastID: result.lastID };
}
export async function updateUser(userId: number, updates: any): Promise<any> {
const fields: string[] = [];
const values: (string | number | null)[] = [];
if (updates.username !== undefined) {
fields.push('username = ?');
values.push(updates.username);
}
if (updates.profilePictureUrl !== undefined) {
fields.push('profile_picture_url = ?');
values.push(updates.profilePictureUrl);
}
if (updates.password !== undefined) {
if (updates.password === null || updates.password === '') {
// Eliminar contraseña
fields.push('password_hash = ?');
values.push(null);
} else {
// Actualizar contraseña
const hash = await bcrypt.hash(updates.password.trim(), SALT_ROUNDS);
fields.push('password_hash = ?');
values.push(hash);
}
}
if (fields.length === 0) {
return { changes: 0, lastID: userId };
}
const setClause = fields.join(', ');
const sql = `UPDATE User SET ${setClause} WHERE id = ?`;
values.push(userId);
return await run(sql, values, USER_DB_NAME);
}
export async function deleteUser(userId: number): Promise<any> {
await run(
`DELETE FROM ListEntry WHERE user_id = ?`,
[userId],
USER_DB_NAME
);
await run(
`DELETE FROM UserIntegration WHERE user_id = ?`,
[userId],
USER_DB_NAME
);
await run(
`DELETE FROM favorites WHERE user_id = ?`,
[userId],
'favorites'
);
const result = await run(
`DELETE FROM User WHERE id = ?`,
[userId],
USER_DB_NAME
);
return result;
}
export async function getAllUsers(): Promise<User[]> {
const sql = `
SELECT
id,
username,
profile_picture_url,
CASE WHEN password_hash IS NOT NULL THEN 1 ELSE 0 END as has_password
FROM User
ORDER BY id
`;
const users = await queryAll(sql, [], USER_DB_NAME);
return users.map((user: any) => ({
id: user.id,
username: user.username,
profile_picture_url: user.profile_picture_url || null,
has_password: !!user.has_password
})) as User[];
}
export async function getUserById(id: number): Promise<User | null> {
const sql = `
SELECT
id,
username,
profile_picture_url,
CASE WHEN password_hash IS NOT NULL THEN 1 ELSE 0 END as has_password
FROM User
WHERE id = ?
`;
const user = await queryOne(sql, [id], USER_DB_NAME);
if (!user) return null;
return {
id: user.id,
username: user.username,
profile_picture_url: user.profile_picture_url || null,
has_password: !!user.has_password
};
}
export async function verifyPassword(userId: number, password: string): Promise<boolean> {
const sql = 'SELECT password_hash FROM User WHERE id = ?';
const user = await queryOne(sql, [userId], USER_DB_NAME);
if (!user || !user.password_hash) {
return false;
}
return await bcrypt.compare(password, user.password_hash);
}
export async function getAniListIntegration(userId: number) {
const sql = `
SELECT anilist_user_id, expires_at
FROM UserIntegration
WHERE user_id = ? AND platform = ?
`;
const row = await queryOne(sql, [userId, "AniList"], USER_DB_NAME);
if (!row) {
return { connected: false };
}
return {
connected: true,
anilistUserId: row.anilist_user_id,
expiresAt: row.expires_at
};
}
export async function removeAniListIntegration(userId: number) {
const sql = `
DELETE FROM UserIntegration
WHERE user_id = ? AND platform = ?
`;
return run(sql, [userId, "AniList"], USER_DB_NAME);
}

View File

@@ -97,7 +97,7 @@ function applyStyles() {
document.documentElement.style.setProperty('--ln-max-width', config.ln.maxWidth + 'px'); document.documentElement.style.setProperty('--ln-max-width', config.ln.maxWidth + 'px');
document.documentElement.style.setProperty('--ln-font-family', config.ln.fontFamily); document.documentElement.style.setProperty('--ln-font-family', config.ln.fontFamily);
document.documentElement.style.setProperty('--ln-text-color', config.ln.textColor); document.documentElement.style.setProperty('--ln-text-color', config.ln.textColor);
document.documentElement.style.setProperty('--bg-base', config.ln.bg); document.documentElement.style.setProperty('--color-bg-base', config.ln.bg);
document.documentElement.style.setProperty('--ln-text-align', config.ln.textAlign); document.documentElement.style.setProperty('--ln-text-align', config.ln.textAlign);
} }

View File

@@ -0,0 +1,102 @@
const Gitea_OWNER = "ItsSkaiya";
const Gitea_REPO = "WaifuBoard";
const CURRENT_VERSION = "v2.0.0-rc.1";
const UPDATE_CHECK_INTERVAL = 5 * 60 * 1000;
let currentVersionDisplay;
let latestVersionDisplay;
let updateToast;
document.addEventListener("DOMContentLoaded", () => {
currentVersionDisplay = document.getElementById("currentVersionDisplay");
latestVersionDisplay = document.getElementById("latestVersionDisplay");
updateToast = document.getElementById("updateToast");
if (currentVersionDisplay) {
currentVersionDisplay.textContent = CURRENT_VERSION;
}
checkForUpdates();
setInterval(checkForUpdates, UPDATE_CHECK_INTERVAL);
});
function showToast(latestVersion) {
if (latestVersionDisplay && updateToast) {
latestVersionDisplay.textContent = latestVersion;
updateToast.classList.add("update-available");
updateToast.classList.remove("hidden");
} else {
console.error(
"Error: Cannot display toast because one or more DOM elements were not found.",
);
}
}
function hideToast() {
if (updateToast) {
updateToast.classList.add("hidden");
updateToast.classList.remove("update-available");
}
}
function isVersionOutdated(versionA, versionB) {
const vA = versionA.replace(/^v/, "").split(".").map(Number);
const vB = versionB.replace(/^v/, "").split(".").map(Number);
for (let i = 0; i < Math.max(vA.length, vB.length); i++) {
const numA = vA[i] || 0;
const numB = vB[i] || 0;
if (numA < numB) return true;
if (numA > numB) return false;
}
return false;
}
async function checkForUpdates() {
console.log(`Checking for updates for ${Gitea_OWNER}/${Gitea_REPO}...`);
const apiUrl = `https://git.waifuboard.app/api/v1/repos/${Gitea_OWNER}/${Gitea_REPO}/releases/latest`;
try {
const response = await fetch(apiUrl, {
method: "GET",
headers: {
Accept: "application/json",
},
});
if (!response.ok) {
if (response.status === 404) {
console.info("No releases found for this repository.");
return;
}
throw new Error(
`Gitea API error: ${response.status} ${response.statusText}`,
);
}
const data = await response.json();
const latestVersion = data.tag_name;
if (!latestVersion) {
console.warn("Release found but no tag_name present");
return;
}
console.log(`Latest Gitea Release: ${latestVersion}`);
if (isVersionOutdated(CURRENT_VERSION, latestVersion)) {
console.warn("Update available!");
showToast(latestVersion);
} else {
console.info("Package is up to date.");
hideToast();
}
} catch (error) {
console.error("Failed to fetch Gitea release:", error);
}
}

View File

@@ -0,0 +1,147 @@
const path = require("path");
const fs = require("fs");
const { chromium } = require("playwright-core");
let browser;
let context;
const BLOCK_LIST = [
"google-analytics", "doubleclick", "facebook", "twitter",
"adsystem", "analytics", "tracker", "pixel", "quantserve", "newrelic"
];
function isPackaged() {
return process.env.IS_PACKAGED === "true";
}
function getChromiumPath() {
if (isPackaged()) {
return path.join(
process.resourcesPath,
"playwright",
"chromium",
"chrome-headless-shell-win64",
"chrome-headless-shell.exe"
);
}
return chromium.executablePath();
}
async function initHeadless() {
if (browser) return;
const exePath = getChromiumPath();
if (!fs.existsSync(exePath)) {
throw new Error("Chromium not found: " + exePath);
}
browser = await chromium.launch({
headless: true,
executablePath: exePath,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu",
"--disable-extensions",
"--disable-background-networking",
"--disable-sync",
"--disable-translate",
"--mute-audio",
"--no-first-run",
"--no-zygote",
]
});
context = await browser.newContext({
userAgent:
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36"
});
}
async function turboScroll(page) {
await page.evaluate(() => {
return new Promise((resolve) => {
let last = 0;
let same = 0;
const timer = setInterval(() => {
const h = document.body.scrollHeight;
window.scrollTo(0, h);
if (h === last) {
same++;
if (same >= 5) {
clearInterval(timer);
resolve();
}
} else {
same = 0;
last = h;
}
}, 20);
});
});
}
async function scrape(url, handler, options = {}) {
const {
waitUntil = "domcontentloaded",
waitSelector = null,
timeout = 10000,
scrollToBottom = false,
renderWaitTime = 0,
loadImages = true
} = options;
if (!browser) await initHeadless();
const page = await context.newPage();
let collectedRequests = [];
await page.route("**/*", (route) => {
const req = route.request();
const rUrl = req.url().toLowerCase();
const type = req.resourceType();
collectedRequests.push({
url: req.url(),
method: req.method(),
resourceType: type
});
if (type === "font" || type === "media" || type === "manifest")
return route.abort();
if (BLOCK_LIST.some(k => rUrl.includes(k)))
return route.abort();
if (!loadImages && (
type === "image" || rUrl.match(/\.(jpg|jpeg|png|gif|webp|svg)$/)
)) return route.abort();
route.continue();
});
await page.goto(url, { waitUntil, timeout });
if (waitSelector) {
try {
await page.waitForSelector(waitSelector, { timeout });
} catch {}
}
if (scrollToBottom) {
await turboScroll(page);
}
if (renderWaitTime > 0) {
await new Promise(r => setTimeout(r, renderWaitTime));
}
const result = await handler(page);
await page.close();
return { result, requests: collectedRequests };
}
async function closeScraper() {
if (context) await context.close();
if (browser) await browser.close();
context = null;
browser = null;
}
module.exports = {
initHeadless,
scrape,
closeScraper
};

Some files were not shown because too many files have changed in this diff Show More