itsskaiya
04f37218de
Removed tabs and moved over to pages Updated the rendering system Fixed multiple pages not loading on scroll and re-rending or not rendering anything or just page 1. Fixed the search bar not taking in spaces for each query Updated how extensions are made Updated how extensions are loaded
102 lines
3.2 KiB
JavaScript
102 lines
3.2 KiB
JavaScript
const fs = require('fs');
|
|
const fetchPath = require.resolve('node-fetch');
|
|
const cheerioPath = require.resolve('cheerio');
|
|
|
|
function peekProperty(filePath, propertyName) {
|
|
try {
|
|
const content = fs.readFileSync(filePath, 'utf-8');
|
|
const regex = new RegExp(`(?:this\\.|^|\\s)${propertyName}\\s*=\\s*["']([^"']+)["']`);
|
|
const match = content.match(regex);
|
|
return match ? match[1] : null;
|
|
} catch (e) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
module.exports = function (availableScrapers, headlessBrowser) {
|
|
|
|
Object.keys(availableScrapers).forEach(name => {
|
|
const scraper = availableScrapers[name];
|
|
|
|
if (!scraper.url) {
|
|
if (scraper.instance && scraper.instance.baseUrl) {
|
|
scraper.url = scraper.instance.baseUrl;
|
|
} else {
|
|
scraper.url = peekProperty(scraper.path, 'baseUrl');
|
|
}
|
|
}
|
|
|
|
if (!scraper.type) {
|
|
if (scraper.instance && scraper.instance.type) {
|
|
scraper.type = scraper.instance.type;
|
|
} else {
|
|
const typeFromFile = peekProperty(scraper.path, 'type');
|
|
if (typeFromFile) {
|
|
console.log(`[API] Recovered type for ${name} via static analysis: ${typeFromFile}`);
|
|
scraper.type = typeFromFile;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
return {
|
|
getSources: () => {
|
|
console.log("[API] Handling getSources request...");
|
|
|
|
const results = Object.keys(availableScrapers).map((name) => {
|
|
const scraper = availableScrapers[name];
|
|
|
|
const typeToReturn = scraper.type || null;
|
|
|
|
console.log(`[API] Processing ${name}: Type found = "${typeToReturn}"`);
|
|
|
|
return {
|
|
name: name,
|
|
url: scraper.url || name,
|
|
type: typeToReturn
|
|
};
|
|
});
|
|
|
|
return results;
|
|
},
|
|
|
|
search: async (event, source, query, page) => {
|
|
const scraperData = availableScrapers[source];
|
|
|
|
if (!scraperData) {
|
|
return { success: false, error: `Source ${source} not found.` };
|
|
}
|
|
|
|
if (!scraperData.instance) {
|
|
console.log(`[LazyLoad] Initializing scraper: ${source}...`);
|
|
try {
|
|
const scraperModule = require(scraperData.path);
|
|
const className = Object.keys(scraperModule)[0];
|
|
const ScraperClass = scraperModule[className];
|
|
|
|
if (!ScraperClass || typeof ScraperClass !== 'function') {
|
|
throw new Error(`File ${scraperData.path} does not export a valid class.`);
|
|
}
|
|
|
|
const instance = new ScraperClass(fetchPath, cheerioPath, headlessBrowser);
|
|
scraperData.instance = instance;
|
|
|
|
if (instance.type) scraperData.type = instance.type;
|
|
if (instance.baseUrl) scraperData.url = instance.baseUrl;
|
|
|
|
} catch (err) {
|
|
console.error(`Failed to lazy load ${source}:`, err);
|
|
return { success: false, error: `Failed to load extension: ${err.message}` };
|
|
}
|
|
}
|
|
|
|
try {
|
|
const results = await scraperData.instance.fetchSearchResult(query, page);
|
|
return { success: true, data: results };
|
|
} catch (err) {
|
|
console.error(`Error during search in ${source}:`, err);
|
|
return { success: false, error: err.message };
|
|
}
|
|
}
|
|
};
|
|
}; |