scrapely 1.2.3 → 1.2.4

package/dist/patch-browser-pool.js

@@ -0,0 +1,827 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setCdpUrl = setCdpUrl;
+exports.getCdpUrlForTests = getCdpUrlForTests;
+exports.resetBrowserProxyWarningsForTests = resetBrowserProxyWarningsForTests;
+exports.applyProxyInfoToCdpUrl = applyProxyInfoToCdpUrl;
+exports.patchBrowserPoolModule = patchBrowserPoolModule;
+exports.patchCrawleeCrawlerModule = patchCrawleeCrawlerModule;
+exports.patchLoadedCrawleeCopies = patchLoadedCrawleeCopies;
+exports.patchRawBrowserType = patchRawBrowserType;
+/**
+ * Redirects every browser launch (Crawlee browser pool, raw playwright /
+ * puppeteer) to the Scrapely remote browser over CDP. The remote browser
+ * never takes a proxy: `proxy_url` is stripped from the CDP URL, custom
+ * (BYOP) proxies are warned about and ignored, and location comes from
+ * `browserConfiguration`, a platform ProxyConfiguration, or the run default
+ * (see applyProxyInfoToCdpUrl).
+ */
+const browser_configuration_js_1 = require("./browser-configuration.js");
+const browser_locations_js_1 = require("./browser-locations.js");
+let _cdpUrl;
+/**
+ * Lazily get CDP URL from environment or cached value.
+ * This allows non-browser actors to work without CDP being set.
+ */
+function getCdpUrl() {
+    return _cdpUrl || process.env.CDP_URL;
+}
+/**
+ * Set CDP URL explicitly (for testing or advanced use cases).
+ * Pass `undefined` to clear the override and fall back to `CDP_URL` env.
+ * Normally the CDP URL is read from CDP_URL env var lazily.
+ */
+function setCdpUrl(url) {
+    _cdpUrl = url;
+}
+/** @internal Exported for tests. */
+function getCdpUrlForTests() {
+    return getCdpUrl();
+}
+/**
+ * Whether the process runs on the Scrapely platform (worker-injected env).
+ */
+function isAtHome() {
+    return !!(process.env.SCRAPELY_IS_AT_HOME || process.env.APIFY_IS_AT_HOME);
+}
+/**
+ * Unpatched builds (browser patch off at build time) keep their browser
+ * binaries; the worker sets SCRAPELY_BROWSER_PATCH=0 so every launch goes to
+ * the original local launcher instead of the remote CDP browser. Read lazily
+ * at launch time.
+ */
+function browserPatchDisabled() {
+    return process.env.SCRAPELY_BROWSER_PATCH === '0';
+}
+/** gVisor lacks crashpad support; ReadonlyRootfs + tmpfs /tmp loses crash dirs. */
+const GVISOR_CHROME_ARGS = [
+    '--disable-breakpad',
+    '--disable-crash-reporter',
+    '--disable-dev-shm-usage',
+    '--crash-dumps-dir=/tmp',
+];
+/** Add gVisor-safe Chrome args to local launch options when running under runsc. */
+function withGVisorArgs(options) {
+    const opts = { ...(options ?? {}) };
+    if (process.env.SCRAPELY_CONTAINER_RUNTIME !== 'runsc')
+        return opts;
+    const args = [...(opts.args ?? [])];
+    for (const arg of GVISOR_CHROME_ARGS) {
+        if (!args.includes(arg))
+            args.push(arg);
+    }
+    opts.args = args;
+    return opts;
+}
+/**
+ * Require CDP_URL for platform browser runs. Local launch is disabled on the
+ * platform unless the build is unpatched (SCRAPELY_BROWSER_PATCH=0) — otherwise
+ * a worker container without CDP_URL is a misconfiguration and must fail
+ * loudly instead of launching Chromium inside the (gVisor) container.
+ */
+function requireCdpUrl() {
+    const cdpUrl = getCdpUrl();
+    if (!cdpUrl) {
+        throw new Error('cdpUrl is required. Local browser launch is disabled — all runs must use the remote browser.');
+    }
+    return cdpUrl;
+}
+const BYOP_IGNORED_WARNING = '[scrapely] Custom proxies are ignored when using the Scrapely remote browser — it routes traffic through its own ' +
+    'integrated proxy. Set the location with browserConfiguration (Actor.createBrowserConfiguration({ countryCode, state, city })) ' +
+    'or ProxyConfiguration countryCode.';
+const GROUPS_SESSION_INFO = '[scrapely] Proxy groups and session IDs do not apply to the Scrapely remote browser; routing is location-based ' +
+    '(proxy_country/proxy_state). Each browser connection is a fresh session.';
+let byopWarned = false;
+let groupsInfoLogged = false;
+let unknownSubdivisionWarned = false;
+/** @internal Reset the once-per-process browser proxy messages (tests only). */
+function resetBrowserProxyWarningsForTests() {
+    byopWarned = false;
+    groupsInfoLogged = false;
+    unknownSubdivisionWarned = false;
+}
+/** Parse a proxy URL, accepting puppeteer-style scheme-less `host:port`. */
+function parseProxyUrl(proxyUrl) {
+    for (const candidate of [proxyUrl, `http://${proxyUrl}`]) {
+        try {
+            const parsed = new URL(candidate);
+            if (parsed.hostname)
+                return parsed;
+        }
+        catch {
+            // try the next form
+        }
+    }
+    return undefined;
+}
+/**
+ * Platform gateway hostname injected by the worker (same vars as
+ * getPlatformProxyEnv). Read directly: this module also loads in SDK-less
+ * processes (base image register hook) and must not pull in @crawlee/core.
+ */
+function platformProxyHostname() {
+    const hostname = (process.env.SCRAPELY_PROXY_HOSTNAME || process.env.APIFY_PROXY_HOSTNAME || '').trim().toLowerCase();
+    return hostname || undefined;
+}
+function isPlatformProxyUrl(proxyUrl) {
+    const platformHost = platformProxyHostname();
+    if (!platformHost)
+        return false;
+    return parseProxyUrl(proxyUrl)?.hostname.toLowerCase() === platformHost;
+}
+/** `scheme://***@host:port` — never log proxy credentials. */
+function redactProxyUrl(proxyUrl) {
+    const parsed = parseProxyUrl(proxyUrl);
+    if (!parsed)
+        return '<unparseable proxy URL>';
+    const userinfo = parsed.username || parsed.password ? '***@' : '';
+    return `${parsed.protocol}//${userinfo}${parsed.host}`;
+}
+/**
+ * Location from the gateway username grammar (`sub,groups-X,session-Y,country-US_CA`).
+ */
+function parseProxyUrlLocation(proxyUrl) {
+    if (!proxyUrl)
+        return {};
+    const parsed = parseProxyUrl(proxyUrl);
+    if (!parsed?.username)
+        return {};
+    let username;
+    try {
+        username = decodeURIComponent(parsed.username);
+    }
+    catch {
+        username = parsed.username;
+    }
+    for (const fragment of username.split(',')) {
+        const match = /^country-([A-Za-z]{2})(?:_([A-Za-z0-9]{1,3}))?$/.exec(fragment.trim());
+        if (match)
+            return { country: match[1].toUpperCase(), subdivision: match[2]?.toUpperCase() };
+    }
+    return {};
+}
+/** Location requested by a platform-gateway proxyInfo; `{}` for none or BYOP. */
+function resolveLaunchLocation(proxyInfo) {
+    const proxyUrl = proxyInfo?.url;
+    if (typeof proxyUrl !== 'string' || !proxyUrl)
+        return {};
+    if (!isPlatformProxyUrl(proxyUrl)) {
+        if (!byopWarned) {
+            byopWarned = true;
+            console.warn(`${BYOP_IGNORED_WARNING} (ignored: ${redactProxyUrl(proxyUrl)})`);
+        }
+        return {};
+    }
+    if ((proxyInfo?.groups?.length || proxyInfo?.sessionId != null) && !groupsInfoLogged) {
+        groupsInfoLogged = true;
+        console.info(GROUPS_SESSION_INFO);
+    }
+    const grammar = parseProxyUrlLocation(proxyUrl);
+    return {
+        country: proxyInfo?.countryCode ?? grammar.country,
+        subdivision: proxyInfo?.subdivisionCode ?? grammar.subdivision,
+    };
+}
+/**
+ * Build the remote browser CDP URL for one browser connection.
+ *
+ * The Scrapely remote browser never takes a proxy: it routes through its own
+ * integrated proxy, selected by location params. Semantics:
+ * - The worker-injected `proxy_country` is the run default; `proxy_url` is
+ *   never sent and is removed if present.
+ * - A platform-gateway `proxyInfo` (crawler ProxyConfiguration) overrides
+ *   `proxy_country` / `proxy_state` (ISO subdivision codes mapped to region
+ *   names). A custom (BYOP) proxy is warned about once and ignored.
+ * - `browserConfiguration` params always win.
+ * - When nothing changes, the CDP URL is returned byte-identical.
+ *
+ * @internal Exported for tests.
+ */
+function applyProxyInfoToCdpUrl(cdpUrl, proxyInfo, browserConfiguration) {
+    if (browserConfiguration != null && !(0, browser_configuration_js_1.isBrowserConfiguration)(browserConfiguration)) {
+        throw new Error('browserConfiguration must be created with Actor.createBrowserConfiguration() or new BrowserConfiguration().');
+    }
+    const url = new URL(cdpUrl);
+    const params = url.searchParams;
+    let changed = false;
+    const setParam = (key, value) => {
+        if (params.get(key) === value)
+            return;
+        params.set(key, value);
+        changed = true;
+    };
+    if (params.has('proxy_url')) {
+        params.delete('proxy_url');
+        changed = true;
+    }
+    const location = resolveLaunchLocation(proxyInfo);
+    const country = location.country?.toUpperCase();
+    let state;
+    if (location.subdivision) {
+        state = (0, browser_locations_js_1.subdivisionToStateName)(country, location.subdivision);
+        if (!state && !unknownSubdivisionWarned) {
+            unknownSubdivisionWarned = true;
+            console.warn(`[scrapely] Subdivision "${country ?? '?'}-${location.subdivision}" has no Scrapely browser mapping; ` +
+                "routing by country only. Use browserConfiguration { state: '<name>' } to target a region.");
+        }
+    }
+    if (country)
+        setParam('proxy_country', country);
+    if (state)
+        setParam('proxy_state', state);
+    if (browserConfiguration != null) {
+        for (const [key, value] of Object.entries(browserConfiguration.toCdpParams())) {
+            setParam(key, value);
+        }
+    }
+    return changed ? url.toString() : cdpUrl;
+}
+/**
+ * @internal Normalize a browser-pool LaunchContext's proxy data into ProxyInfoLike.
+ *
+ * Crawlee's browser crawler sets both `launchContext.proxyInfo` (the ProxyInfo
+ * object, `.url` included) and `launchContext.proxyUrl` (the full proxy URL)
+ * per browser launch. The URL fallback covers callers that only provide the URL.
+ */
+function proxyInfoFromLaunchContext(launchContext) {
+    const info = launchContext?.proxyInfo;
+    if (typeof info?.url === 'string' && info.url) {
+        return {
+            url: info.url,
+            countryCode: info.countryCode,
+            subdivisionCode: info.subdivisionCode,
+            groups: info.groups,
+            sessionId: info.sessionId,
+        };
+    }
+    const proxyUrl = launchContext?.proxyUrl;
+    if (typeof proxyUrl === 'string' && proxyUrl)
+        return { url: proxyUrl };
+    return undefined;
+}
+/**
+ * Resolve Playwright's chromium BrowserType for CDP connect.
+ *
+ * Crawlee's PlaywrightPlugin.library is a BrowserType (chromium|firefox|webkit).
+ * Remote browser is always Chromium CDP, so firefox/webkit/camoufox launchers
+ * are redirected to chromium.connectOverCDP — never denied, never local launch.
+ */
+function getPlaywrightChromium(library) {
+    // Full playwright module passed as library (has .chromium)
+    if (library?.chromium && typeof library.chromium.connectOverCDP === 'function') {
+        return library.chromium;
+    }
+    // BrowserType that is already chromium
+    if (typeof library?.connectOverCDP === 'function' &&
+        typeof library?.name === 'function' &&
+        library.name() === 'chromium') {
+        return library;
+    }
+    // firefox / webkit / unknown BrowserType → force chromium from playwright
+    // Prefer playwright, fall back to playwright-core (images may ship either).
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const pw = require('playwright');
+        if (pw?.chromium?.connectOverCDP) {
+            return pw.chromium;
+        }
+    }
+    catch {
+        // ignore
+    }
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const pwCore = require('playwright-core');
+        if (pwCore?.chromium?.connectOverCDP) {
+            return pwCore.chromium;
+        }
+    }
+    catch {
+        // ignore
+    }
+    // Last resort: if library has connectOverCDP, use it (may be chromium-shaped)
+    if (typeof library?.connectOverCDP === 'function') {
+        return library;
+    }
+    throw new Error('Unable to resolve Playwright chromium for remote CDP connect. Install playwright or playwright-core.');
+}
+/**
+ * Redirect a PlaywrightPlugin class's `_launch` to remote Chromium CDP
+ * (chromium / firefox / webkit / camoufox) on the platform; a normal local
+ * launch when running locally without CDP_URL. Idempotent per class.
+ */
+function patchPlaywrightPluginClass(PlaywrightPlugin) {
+    const proto = PlaywrightPlugin?.prototype;
+    if (!proto || typeof proto._launch !== 'function' || proto._scrapelyCdpPatched)
+        return;
+    const origPlaywrightLaunch = proto._launch;
+    proto._launch = async function (launchContext) {
+        if (browserPatchDisabled() || (!getCdpUrl() && !isAtHome())) {
+            return origPlaywrightLaunch.call(this, launchContext);
+        }
+        const cdpUrl = applyProxyInfoToCdpUrl(requireCdpUrl(), proxyInfoFromLaunchContext(launchContext), launchContext?.browserConfiguration);
+        const chromium = getPlaywrightChromium(this.library);
+        return chromium.connectOverCDP(cdpUrl);
+    };
+    proto._scrapelyCdpPatched = true;
+}
+/**
+ * Redirect a PuppeteerPlugin class's `_launch` to remote Chromium CDP via
+ * browserWSEndpoint on the platform; a normal local launch when running
+ * locally without CDP_URL. Idempotent per class.
+ */
+function patchPuppeteerPluginClass(PuppeteerPlugin) {
+    const proto = PuppeteerPlugin?.prototype;
+    if (!proto || typeof proto._launch !== 'function' || proto._scrapelyCdpPatched)
+        return;
+    const origPuppeteerLaunch = proto._launch;
+    proto._launch = async function (launchContext) {
+        if (browserPatchDisabled() || (!getCdpUrl() && !isAtHome())) {
+            return origPuppeteerLaunch.call(this, launchContext);
+        }
+        const cdpUrl = applyProxyInfoToCdpUrl(requireCdpUrl(), proxyInfoFromLaunchContext(launchContext), launchContext?.browserConfiguration);
+        const launcher = this.library;
+        if (!launcher || typeof launcher.connect !== 'function') {
+            throw new Error('Puppeteer library is missing connect(). Install puppeteer or puppeteer-core.');
+        }
+        return launcher.connect({ browserWSEndpoint: cdpUrl });
+    };
+    proto._scrapelyCdpPatched = true;
+}
+/**
+ * Patch the browser plugins exported by one loaded copy of
+ * `@crawlee/browser-pool` (or one of its plugin files). Used for the SDK's own
+ * copy (eager pass below) and for every actor-local copy (Module._load hook),
+ * since an actor that installs its own Crawlee resolves a different instance.
+ *
+ * @internal Exported for tests.
+ */
+function patchBrowserPoolModule(mod) {
+    if (!mod || (typeof mod !== 'object' && typeof mod !== 'function'))
+        return;
+    patchPlaywrightPluginClass(mod.PlaywrightPlugin);
+    patchPuppeteerPluginClass(mod.PuppeteerPlugin);
+    patchControllerProxyOptions(mod.PlaywrightController);
+    patchControllerProxyOptions(mod.PuppeteerController);
+}
+/**
+ * With incognito pages Crawlee turns the launch proxy into per-context proxy
+ * options. On the remote browser that would route pages through a custom
+ * proxy (or double-proxy the gateway), so no per-page proxy is set while
+ * connected over CDP.
+ */
+function patchControllerProxyOptions(Controller) {
+    const proto = Controller?.prototype;
+    if (!proto || typeof proto.normalizeProxyOptions !== 'function' || proto._scrapelyProxyOptionsPatched)
+        return;
+    const origNormalize = proto.normalizeProxyOptions;
+    proto.normalizeProxyOptions = function (...args) {
+        if (!browserPatchDisabled() && getCdpUrl())
+            return {};
+        return origNormalize.apply(this, args);
+    };
+    proto._scrapelyProxyOptionsPatched = true;
+}
+// ─── Crawler `browserConfiguration` option ──────────────────────────────────
+//
+// Crawlee validates crawler options with ow.exactShape, and every crawler
+// class copies its parent's optionsShape at module load, so the option is
+// added to each class's own shape: BasicCrawler (browser crawlers forward
+// unknown keys to it) and every browser crawler. HTTP crawlers never accept
+// it — the key is removed from their (possibly copied) shapes so passing it
+// there fails with Crawlee's normal unknown-option error.
+//
+// The predicate reuses the class's own `sessionPoolOptions` predicate
+// (ow.optional.object): an ow predicate from a different ow copy is not
+// recognized as a predicate by exactShape.
+//
+// Acceptance is never gated on CDP_URL / SCRAPELY_BROWSER_PATCH — only the
+// `_launch` redirect is — so the same actor code runs locally unchanged.
+function isCrawlerClass(value) {
+    return typeof value === 'function' && !!value.optionsShape && typeof value.optionsShape === 'object';
+}
+function patchCrawlerClass(CrawlerClass) {
+    const shape = CrawlerClass.optionsShape;
+    if (!Object.prototype.hasOwnProperty.call(CrawlerClass, 'optionsShape'))
+        return;
+    const isBrowserCrawler = 'launchContext' in shape;
+    const isBasicCrawler = CrawlerClass.name === 'BasicCrawler';
+    if (isBrowserCrawler || isBasicCrawler) {
+        if (!('browserConfiguration' in shape) && shape.sessionPoolOptions) {
+            shape.browserConfiguration = shape.sessionPoolOptions;
+        }
+    }
+    else if ('browserConfiguration' in shape) {
+        delete shape.browserConfiguration;
+    }
+    const proto = CrawlerClass.prototype;
+    if (proto &&
+        Object.prototype.hasOwnProperty.call(proto, '_extendLaunchContext') &&
+        typeof proto._extendLaunchContext === 'function' &&
+        !proto._scrapelyBrowserConfigPatched) {
+        const origExtend = proto._extendLaunchContext;
+        proto._extendLaunchContext = async function (pageId, launchContext) {
+            await origExtend.call(this, pageId, launchContext);
+            const browserConfiguration = this.options?.browserConfiguration;
+            if (browserConfiguration != null && launchContext && launchContext.browserConfiguration === undefined) {
+                if (typeof launchContext.extend === 'function') {
+                    launchContext.extend({ browserConfiguration });
+                }
+                else {
+                    launchContext.browserConfiguration = browserConfiguration;
+                }
+            }
+        };
+        proto._scrapelyBrowserConfigPatched = true;
+    }
+}
+/**
+ * Add the `browserConfiguration` option to the crawler classes exported by
+ * one loaded Crawlee module (package index or crawler file). Idempotent.
+ *
+ * @internal Exported for tests.
+ */
+function patchCrawleeCrawlerModule(mod) {
+    if (!mod || (typeof mod !== 'object' && typeof mod !== 'function'))
+        return;
+    const candidates = isCrawlerClass(mod) ? [mod] : [];
+    for (const key of Object.keys(mod)) {
+        let value;
+        try {
+            value = mod[key];
+        }
+        catch {
+            continue;
+        }
+        if (isCrawlerClass(value))
+            candidates.push(value);
+    }
+    for (const CrawlerClass of candidates)
+        patchCrawlerClass(CrawlerClass);
+}
+const CRAWLER_PACKAGES = 'basic|browser|playwright|puppeteer|http|cheerio|jsdom|linkedom';
+/** Requests that load (a part of) some copy of a Crawlee crawler package. */
+const CRAWLER_REQUEST_RE = new RegExp(`^(@crawlee/(${CRAWLER_PACKAGES})|crawlee)$` +
+    `|[/\\\\]@crawlee[/\\\\](${CRAWLER_PACKAGES})[/\\\\]index\\.js$` +
+    `|[/\\\\]node_modules[/\\\\]crawlee[/\\\\]index\\.js$` +
+    '|(^|[/\\\\])internals[/\\\\][a-z-]+-crawler(\\.js)?$');
+/** Cached module filenames of Crawlee crawler packages (any copy). */
+const CRAWLER_FILENAME_RE = new RegExp(`[/\\\\](@crawlee[/\\\\](${CRAWLER_PACKAGES})|crawlee)[/\\\\](internals[/\\\\][a-z-]+-crawler\\.js|index\\.js)$`);
+/**
+ * Patch Crawlee copies that loaded before this module (e.g. an actor that
+ * imports `crawlee` before `scrapely` in a local run). Copies loaded later are
+ * patched by the Module._load hook.
+ *
+ * @internal Exported for tests.
+ */
+function patchLoadedCrawleeCopies() {
+    const cache = require.cache ?? {};
+    for (const [filename, cached] of Object.entries(cache)) {
+        if (cached?.exports && CRAWLER_FILENAME_RE.test(filename)) {
+            patchCrawleeCrawlerModule(cached.exports);
+        }
+    }
+}
+try {
+    // Use Crawlee's ow (0.28.x CJS). Do not import top-level ow@1.x (ESM-only).
+    const owModule = require(require.resolve('ow', { paths: [require.resolve('@crawlee/browser')] }));
+    const ow = owModule.default ?? owModule;
+    // Playwright + Puppeteer plugins of the SDK's own @crawlee/browser-pool copy.
+    // Actor-local copies are patched by the Module._load hook further down.
+    try {
+        patchBrowserPoolModule(require('@crawlee/browser-pool'));
+    }
+    catch {
+        // @crawlee/browser-pool not installed — skip plugin patches
+    }
+    try {
+        const launcherPath = require.resolve('@crawlee/playwright').replace(/[/\\][^/\\]+$/, '') +
+            '/internals/playwright-launcher.js';
+        const { PlaywrightLauncher } = require(launcherPath);
+        PlaywrightLauncher.optionsShape.cdpUrl = ow.optional.string;
+    }
+    catch {
+        // @crawlee/playwright not installed — skip Playwright patches
+    }
+    try {
+        const puppeteerLauncherPath = require.resolve('@crawlee/puppeteer').replace(/[/\\][^/\\]+$/, '') +
+            '/internals/puppeteer-launcher.js';
+        const { PuppeteerLauncher } = require(puppeteerLauncherPath);
+        PuppeteerLauncher.optionsShape.cdpUrl = ow.optional.string;
+    }
+    catch {
+        // @crawlee/puppeteer not installed — plugin patch above is enough
+    }
+    // Patch @crawlee/browser BrowserLauncher when available
+    const browserPkg = require.resolve('@crawlee/browser');
+    const browserLauncherPath = browserPkg.replace(/[/\\][^/\\]+$/, '') + '/internals/browser-launcher.js';
+    const { BrowserLauncher } = require(browserLauncherPath);
+    BrowserLauncher.optionsShape.cdpUrl = ow.optional.string;
+    // Patch createLaunchOptions to inject gVisor-safe Chrome args for any residual
+    // local launch path. Playwright/Puppeteer pool launches use CDP and skip this.
+    // Only applies when SCRAPELY_CONTAINER_RUNTIME=runsc (gVisor sandbox).
+    const origCreateLaunchOptions = BrowserLauncher.prototype.createLaunchOptions;
+    BrowserLauncher.prototype.createLaunchOptions = function () {
+        const launchOptions = origCreateLaunchOptions.call(this);
+        // Inject crashpad-disabling args when running under gVisor
+        // gVisor doesn't support Chrome's crashpad_handler process
+        // Also needed with ReadonlyRootfs + tmpfs /tmp where /tmp/.X11-unix
+        // and crashpad database dir don't persist from the base image.
+        if (process.env.SCRAPELY_CONTAINER_RUNTIME === 'runsc') {
+            launchOptions.args = launchOptions.args || [];
+            for (const arg of GVISOR_CHROME_ARGS) {
+                if (!launchOptions.args.includes(arg)) {
+                    launchOptions.args.push(arg);
+                }
+            }
+        }
+        return launchOptions;
+    };
+}
+catch {
+    // @crawlee/browser not installed — skip browser pool patches (e.g. cheerio-only actors)
+}
+/**
+ * Split our `browserConfiguration` option off raw launch options. The options
+ * object is returned as-is when the key is absent.
+ */
+function takeBrowserConfiguration(options) {
+    if (!options || typeof options !== 'object' || !('browserConfiguration' in options)) {
+        return { options: options ?? {}, browserConfiguration: undefined };
+    }
+    const { browserConfiguration, ...rest } = options;
+    return { options: rest, browserConfiguration };
+}
+/** Options that only make sense for a local launch (no remote-CDP equivalent). */
+const RAW_LAUNCH_ONLY_KEYS = new Set([
+    'headless',
+    'args',
+    'ignoreDefaultArgs',
+    'ignoreArgs',
+    'executablePath',
+    'channel',
+    'env',
+    'devtools',
+    'chromiumSandbox',
+    'firefoxUserPrefs',
+    'proxy',
+    'tracesDir',
+    'waitForExtensions',
+    'userDataDir',
+    'user_data_dir',
+    'downloadsPath',
+]);
+function composeRawProxyUrl(proxy) {
+    if (!proxy?.server)
+        return undefined;
+    if (proxy.username) {
+        try {
+            const url = new URL(proxy.server);
+            url.username = proxy.username;
+            url.password = proxy.password ?? '';
+            return url.toString();
+        }
+        catch {
+            return proxy.server;
+        }
+    }
+    return proxy.server;
+}
+function rawLaunchCdpUrl(options, browserConfiguration) {
+    const cdpUrl = getCdpUrl();
+    if (!cdpUrl)
+        return undefined;
+    const proxyUrl = composeRawProxyUrl(options?.proxy);
+    return applyProxyInfoToCdpUrl(cdpUrl, proxyUrl ? { url: proxyUrl } : undefined, browserConfiguration);
+}
+function rawConnectOptions(options) {
+    const out = {};
+    if (options.timeout !== undefined)
+        out.timeout = options.timeout;
+    if (options.slowMo !== undefined)
+        out.slowMo = options.slowMo;
+    return out;
+}
+function warnDroppedRawOptions(kind, name, options) {
+    const dropped = Object.keys(options).filter((k) => k !== 'timeout' && k !== 'slowMo');
+    if (dropped.length > 0) {
+        console.warn(`[scrapely] playwright ${name}.${kind} redirected to the remote CDP browser; ` +
+            `local launch options ignored: ${dropped.join(', ')} ` +
+            '(the remote browser manages headless mode, binary and flags).');
+    }
+}
+/** @internal Exported for tests. */
+function patchRawBrowserType(chromium, browserType, name) {
+    if (!browserType || typeof browserType.launch !== 'function')
+        return;
+    if (browserType._scrapelyRawCdpPatched)
+        return;
+    const origLaunch = browserType.launch;
+    const origLpc = browserType.launchPersistentContext;
+    browserType.launch = async function rawLaunchPatched(launchOptions = {}) {
+        const { options, browserConfiguration } = takeBrowserConfiguration(launchOptions);
+        if (browserPatchDisabled())
+            return origLaunch.call(this, withGVisorArgs(options));
+        const cdpUrl = rawLaunchCdpUrl(options, browserConfiguration);
+        if (!cdpUrl) {
+            if (!isAtHome())
+                return origLaunch.call(this, options);
+            throw new Error('cdpUrl is required. Local browser launch is disabled — all runs must use the remote browser.');
+        }
+        warnDroppedRawOptions('launch', name, options);
+        return chromium.connectOverCDP(cdpUrl, rawConnectOptions(options));
+    };
+    if (typeof origLpc === 'function') {
+        browserType.launchPersistentContext = async function rawLpcPatched(...lpcArgs) {
+            // Playwright signature: launchPersistentContext(userDataDir, options);
+            // an options-only call is accepted too.
+            const optionsIndex = typeof lpcArgs[0] === 'string' ? 1 : 0;
+            const userDataDir = optionsIndex === 1 ? lpcArgs[0] : undefined;
+            const { options, browserConfiguration } = takeBrowserConfiguration(lpcArgs[optionsIndex]);
+            const withOptions = (opts) => optionsIndex === 1 ? [userDataDir, opts] : [opts];
+            if (browserPatchDisabled())
+                return origLpc.call(this, ...withOptions(withGVisorArgs(options)));
+            const cdpUrl = rawLaunchCdpUrl(options, browserConfiguration);
+            if (!cdpUrl) {
+                if (!isAtHome()) {
+                    return origLpc.call(this, ...withOptions(lpcArgs[optionsIndex] === undefined ? undefined : options));
+                }
+                throw new Error('cdpUrl is required. Local browser launch is disabled — all runs must use the remote browser.');
+            }
+            const profileDir = userDataDir ?? options.userDataDir;
+            if (profileDir) {
+                console.warn(`[scrapely] playwright ${name}.launchPersistentContext(userDataDir=${profileDir}) redirected to the ` +
+                    'remote CDP browser — persistent profiles have no remote equivalent; a fresh context is returned.');
+            }
+            warnDroppedRawOptions('launchPersistentContext', name, options);
+            const browser = await chromium.connectOverCDP(cdpUrl, rawConnectOptions(options));
+            const contextOptions = {};
+            for (const [k, v] of Object.entries(options)) {
+                if (!RAW_LAUNCH_ONLY_KEYS.has(k) && k !== 'timeout' && k !== 'slowMo')
+                    contextOptions[k] = v;
+            }
+            return browser.newContext(contextOptions);
+        };
+    }
+    browserType._scrapelyRawCdpPatched = true;
+}
+/** Patch the chromium/firefox/webkit BrowserTypes of a playwright module. */
+function patchRawPlaywrightModule(pw) {
+    if (!pw?.chromium || typeof pw.chromium.connectOverCDP !== 'function')
+        return;
+    for (const name of ['chromium', 'firefox', 'webkit']) {
+        patchRawBrowserType(pw.chromium, pw[name], name);
+    }
+}
+/**
+ * Patch a raw puppeteer launcher: the default export (also exposed as module
+ * props). Both the full puppeteer package and puppeteer-core ship the same
+ * launcher shape (launch/connect) — patch whichever was loaded.
+ */
+function patchRawPuppeteerModule(mod, moduleName) {
+    const launcher = mod?.default ?? mod;
+    if (typeof launcher?.launch !== 'function' || launcher._scrapelyRawCdpPatched) {
+        return;
+    }
+    const origLaunch = launcher.launch.bind(launcher);
+    launcher.launch = async function rawPuppeteerLaunchPatched(launchOptions = {}) {
+        const { options, browserConfiguration } = takeBrowserConfiguration(launchOptions);
+        if (browserPatchDisabled())
+            return origLaunch(withGVisorArgs(options));
+        const cdpUrl = rawLaunchCdpUrl(options, browserConfiguration);
+        if (!cdpUrl) {
+            if (!isAtHome())
+                return origLaunch(options);
+            throw new Error('cdpUrl is required. Local browser launch is disabled — all runs must use the remote browser.');
+        }
+        const dropped = Object.keys(options).filter((k) => k !== 'timeout' && k !== 'slowMo' && k !== 'protocol');
+        if (dropped.length > 0) {
+            console.warn(`[scrapely] ${moduleName}.launch redirected to the remote CDP browser; ` +
+                `local launch options ignored: ${dropped.join(', ')}.`);
+        }
+        return launcher.connect({ browserWSEndpoint: cdpUrl, ...rawConnectOptions(options) });
+    };
+    launcher._scrapelyRawCdpPatched = true;
+}
+/**
+ * Eager pass: patch whatever playwright/puppeteer is resolvable from this
+ * module's own dependency tree right now (the SDK tree under
+ * /opt/scrapely/node_modules in the base image). Actor-local installs are
+ * covered by the Module._load hook below — a bare require() from this file
+ * can never resolve them.
+ */
+function patchSdkTreeRawModules() {
+    for (const pwModuleName of ['playwright', 'playwright-core']) {
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            patchRawPlaywrightModule(require(pwModuleName));
+        }
+        catch {
+            // not installed in the SDK tree — actor-local copies still get the hook
+        }
+    }
+    // Raw puppeteer API: default export launcher (also exposed as module props).
+    // Both the full puppeteer package and puppeteer-core ship the same launcher
+    // shape (launch/connect) — patch whichever is installed.
+    for (const puppeteerModuleName of ['puppeteer', 'puppeteer-core']) {
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            patchRawPuppeteerModule(require(puppeteerModuleName), puppeteerModuleName);
+        }
+        catch {
+            // not installed in the SDK tree — actor-local copies still get the hook
+        }
+    }
+}
+/**
+ * Apply the raw patches to a module coming back from the hooked loader.
+ * `request` is the specifier as written by the caller, `loaded` is whatever
+ * instance it actually resolved to, and `loadRoot` resolves a package root
+ * through the original loader (no hook recursion) for subpath requests.
+ */
+/**
+ * Requests that load (a part of) some copy of @crawlee/browser-pool:
+ * - the bare package name (require from @crawlee/playwright etc.),
+ * - an absolute path to its index.js (ESM → CJS interop loads by filename),
+ * - the relative plugin files the package's own index.js requires.
+ */
+const BROWSER_POOL_REQUEST_RE = /^@crawlee\/browser-pool$|[/\\]@crawlee[/\\]browser-pool[/\\]index\.js$|(^|[/\\])(playwright|puppeteer)-plugin(\.js)?$/;
+function patchLoadedRawModule(request, loaded, loadRoot) {
+    if (typeof request !== 'string')
+        return;
+    // Crawlee browser pool — the actor's own copy (e.g. /usr/src/app/node_modules)
+    // is a different instance from the SDK's, so its plugins need the CDP patch too.
+    if (BROWSER_POOL_REQUEST_RE.test(request)) {
+        patchBrowserPoolModule(loaded);
+        return;
+    }
+    if (CRAWLER_REQUEST_RE.test(request)) {
+        patchCrawleeCrawlerModule(loaded);
+        return;
+    }
+    if (request === 'playwright' || request === 'playwright-core') {
+        patchRawPlaywrightModule(loaded);
+        return;
+    }
+    if (request === 'puppeteer' || request === 'puppeteer-core') {
+        patchRawPuppeteerModule(loaded, request);
+        return;
+    }
+    // BrowserType subpath loads, e.g. require('playwright/chromium') — the
+    // export is the BrowserType itself, not the package root. Resolve the root
+    // for its chromium (the connectOverCDP anchor), patch the root, then the
+    // subpath export. The launch() shape check ignores deep imports into the
+    // package that are not BrowserTypes.
+    if (!request.startsWith('playwright/') && !request.startsWith('playwright-core/'))
+        return;
+    const subpath = /^(playwright|playwright-core)\/(.+)$/.exec(request);
+    if (subpath && typeof loaded?.launch === 'function') {
+        try {
+            const root = loadRoot(subpath[1]);
+            patchRawPlaywrightModule(root);
+            if (root?.chromium)
+                patchRawBrowserType(root.chromium, loaded, subpath[2]);
+        }
+        catch {
+            // package root unresolvable — no chromium anchor to connect through
+        }
+    }
+}
+/**
+ * Hook the CommonJS loader so the raw patches reach actor-local installs.
+ *
+ * The eager pass above can only see this module's own dependency tree; an
+ * actor bundling playwright/puppeteer in its workdir resolves a different,
+ * unpatched instance. Module._load is the chokepoint every require() passes
+ * through AFTER resolution — including CJS packages imported from ESM, which
+ * Node funnels through the CJS loader — so patching here covers whichever
+ * copy (SDK tree or actor workdir) the request actually resolved to. Chained
+ * onto the current _load (composes with tsx's hook in dev) and idempotent.
+ */
+function installRawModuleLoadHook() {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const Module = require('module');
+    const origLoad = Module._load;
+    if (typeof origLoad !== 'function' || origLoad._scrapelyRawCdpHook)
+        return;
+    const hookedLoad = function (request, parent, isMain) {
+        const loaded = origLoad.apply(this, [request, parent, isMain]);
+        try {
+            patchLoadedRawModule(request, loaded, (name) => origLoad(name, parent, isMain));
+        }
+        catch (err) {
+            // Never break the actor's module loading — warn and hand back the
+            // unpatched module rather than crashing the process.
+            console.warn(`[scrapely] CDP patch failed for '${String(request)}':`, err?.message);
+        }
+        return loaded;
+    };
+    hookedLoad._scrapelyRawCdpHook = true;
+    Module._load = hookedLoad;
+}
+function applyRawBrowserPatches() {
+    patchSdkTreeRawModules();
+    installRawModuleLoadHook();
+    patchLoadedCrawleeCopies();
+}
+applyRawBrowserPatches();