openhands
af8b59e024
puppeteer-extra cannot be statically imported in Next.js server bundles. Using dynamic import() so puppeteer is only loaded at runtime, not at build time.
92 lines
2.2 KiB
TypeScript
92 lines
2.2 KiB
TypeScript
const UA =
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) " +
|
|
"Chrome/125.0.0.0 Safari/537.36";
|
|
|
|
let browser: any = null;
|
|
|
|
async function getBrowser() {
|
|
if (browser) return browser;
|
|
const puppeteer = await import("puppeteer-extra");
|
|
const StealthPlugin = await import("puppeteer-extra-plugin-stealth");
|
|
puppeteer.default.use(StealthPlugin.default());
|
|
browser = await puppeteer.default.launch({
|
|
headless: true,
|
|
args: [
|
|
"--no-sandbox",
|
|
"--disable-setuid-sandbox",
|
|
"--disable-web-security",
|
|
"--disable-blink-features=AutomationControlled",
|
|
"--user-agent=" + UA,
|
|
],
|
|
});
|
|
return browser;
|
|
}
|
|
|
|
export async function fetchWithCfBypass(
|
|
url: string,
|
|
timeout = 30000,
|
|
): Promise<string> {
|
|
const b = await getBrowser();
|
|
const page = await b.newPage();
|
|
await page.setUserAgent(UA);
|
|
await page.setDefaultTimeout(timeout);
|
|
|
|
try {
|
|
const response = await page.goto(url, {
|
|
waitUntil: "networkidle2",
|
|
timeout,
|
|
});
|
|
|
|
const status = response?.status() ?? 0;
|
|
const body = await page.content();
|
|
|
|
const isCfChallenge =
|
|
body.includes("Attention Required") ||
|
|
body.includes("Checking your browser") ||
|
|
body.includes("Just a moment") ||
|
|
body.includes("cf-challenge") ||
|
|
status === 403;
|
|
|
|
if (isCfChallenge) {
|
|
await page.waitForFunction(
|
|
() =>
|
|
!document.body.innerHTML.includes("Attention Required") &&
|
|
!document.body.innerHTML.includes("Checking your browser") &&
|
|
!document.body.innerHTML.includes("Just a moment"),
|
|
{ timeout: 10000 },
|
|
);
|
|
}
|
|
|
|
const finalBody = await page.content();
|
|
return finalBody;
|
|
} finally {
|
|
await page.close();
|
|
}
|
|
}
|
|
|
|
export async function fetchJsonWithCfBypass<T = unknown>(
|
|
url: string,
|
|
timeout = 30000,
|
|
): Promise<T> {
|
|
const html = await fetchWithCfBypass(url, timeout);
|
|
const match = html.match(
|
|
/<pre[^>]*class="[^"]*json[^"]*"[^>]*>([\s\S]*?)<\/pre>/i,
|
|
);
|
|
if (match) {
|
|
return JSON.parse(match[1].trim()) as T;
|
|
}
|
|
try {
|
|
return JSON.parse(html) as T;
|
|
} catch {
|
|
throw new Error(`cf-fetch: could not parse JSON from ${url}`);
|
|
}
|
|
}
|
|
|
|
export async function closeCfBrowser(): Promise<void> {
|
|
if (browser) {
|
|
await browser.close();
|
|
browser = null;
|
|
}
|
|
}
|