scrapely 1.2.26 → 1.2.27

package/dist/lazy-exports.js

@@ -0,0 +1,132 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LAZY_EXPORTS = void 0;
+exports.loadLazyExport = loadLazyExport;
+exports.defineLazyExports = defineLazyExports;
+exports.lazyExport = lazyExport;
+/**
+ * Public SDK names that load heavy packages: Crawlee's crawlers (jsdom,
+ * linkedom, cheerio, the browser pool, playwright) and scrapely-client.
+ * Importing the SDK used to load all of them, ~4 s at 1 core under gVisor;
+ * they now load on first access.
+ *
+ * - CommonJS (`require('scrapely').PlaywrightCrawler`): a getter on the
+ *   package exports that requires the package once, then becomes a plain value.
+ * - ESM (`import { PlaywrightCrawler } from 'scrapely'` / 'apify'): index.mts
+ *   exports a lazy stand-in per name (see lazyExport). Node reads every
+ *   detected export of a CommonJS module when ESM imports it, so these getters
+ *   are deliberately defined in a form cjs-module-lexer does not detect.
+ *
+ * Loaded by the SDK entry: keep this file free of heavy imports.
+ */
+const client_loader_js_1 = require("./client-loader.js");
+/** name → [package, export]. */
+exports.LAZY_EXPORTS = {
+    PlaywrightCrawler: ['@crawlee/playwright', 'PlaywrightCrawler'],
+    PuppeteerCrawler: ['@crawlee/puppeteer', 'PuppeteerCrawler'],
+    CheerioCrawler: ['@crawlee/cheerio', 'CheerioCrawler'],
+    HttpCrawler: ['@crawlee/http', 'HttpCrawler'],
+    BasicCrawler: ['@crawlee/basic', 'BasicCrawler'],
+    BrowserCrawler: ['@crawlee/browser', 'BrowserCrawler'],
+    JSDOMCrawler: ['@crawlee/jsdom', 'JSDOMCrawler'],
+    LinkeDOMCrawler: ['@crawlee/linkedom', 'LinkeDOMCrawler'],
+    playwrightUtils: ['@crawlee/playwright', 'playwrightUtils'],
+    puppeteerUtils: ['@crawlee/puppeteer', 'puppeteerUtils'],
+    ApifyClient: ['scrapely-client', 'ApifyClient'],
+    USES_PUSH_DATA_INTERCEPTION: ['scrapely-client', 'USES_PUSH_DATA_INTERCEPTION'],
+    pushDataChargingContext: ['scrapely-client', 'pushDataChargingContext'],
+};
+/** Resolve one lazy export (loads its package on first call). */
+function loadLazyExport(name) {
+    const entry = exports.LAZY_EXPORTS[name];
+    if (!entry)
+        throw new Error(`Unknown lazy export: ${name}`);
+    if (entry[0] === 'scrapely-client')
+        return (0, client_loader_js_1.scrapelyClient)()[entry[1]];
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    return require(entry[0])[entry[1]];
+}
+/** Define every lazy export as a self-replacing getter on a CommonJS exports object. */
+function defineLazyExports(target) {
+    for (const name of Object.keys(exports.LAZY_EXPORTS)) {
+        Object.defineProperty(target, name, {
+            enumerable: true,
+            configurable: true,
+            get() {
+                const value = loadLazyExport(name);
+                Object.defineProperty(target, name, { value, enumerable: true, configurable: true, writable: false });
+                return value;
+            },
+        });
+    }
+}
+/**
+ * ESM stand-in for a lazy export: a Proxy that loads the real value on first
+ * use and forwards everything to it. For classes it is constructible and
+ * extendable (`new X()`, `class Y extends X`, `instanceof X`, statics); for
+ * objects (playwrightUtils, ...) property access works as usual. The one
+ * visible difference: the stand-in is not `===` the package's own export.
+ */
+function lazyExport(name, kind) {
+    let loaded = false;
+    let real;
+    const load = () => {
+        if (!loaded) {
+            real = loadLazyExport(name);
+            loaded = true;
+        }
+        return real;
+    };
+    const boundMethods = new Map();
+    const bound = (fn, self) => {
+        let b = boundMethods.get(fn);
+        if (!b) {
+            b = fn.bind(self);
+            boundMethods.set(fn, b);
+        }
+        return b;
+    };
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    const target = kind === 'class' ? function LazyExport() { } : {};
+    const proxy = new Proxy(target, {
+        get(_t, prop, receiver) {
+            const value = load();
+            const result = Reflect.get(value, prop, receiver === proxy ? value : receiver);
+            // Object stand-ins: methods run on the real object (private fields,
+            // internal slots), e.g. pushDataChargingContext.run().
+            if (kind === 'object' && typeof result === 'function')
+                return bound(result, value);
+            return result;
+        },
+        set(_t, prop, newValue, receiver) {
+            const value = load();
+            return Reflect.set(value, prop, newValue, receiver === proxy ? value : receiver);
+        },
+        has: (_t, prop) => Reflect.has(load(), prop),
+        ownKeys: () => {
+            const keys = Reflect.ownKeys(load());
+            if (kind === 'class' && !keys.includes('prototype'))
+                keys.push('prototype');
+            return keys;
+        },
+        getOwnPropertyDescriptor(_t, prop) {
+            const desc = Reflect.getOwnPropertyDescriptor(load(), prop);
+            if (!desc)
+                return undefined;
+            // Stay compatible with the stand-in target (Proxy invariants): only
+            // the target function's own `prototype` may be non-configurable.
+            if (kind === 'class' && prop === 'prototype')
+                return { ...desc, writable: true, configurable: false };
+            return { ...desc, configurable: true };
+        },
+        defineProperty: (_t, prop, desc) => Reflect.defineProperty(load(), prop, desc),
+        deleteProperty: (_t, prop) => Reflect.deleteProperty(load(), prop),
+        getPrototypeOf: () => Reflect.getPrototypeOf(load()),
+        construct(_t, args, newTarget) {
+            const value = load();
+            return Reflect.construct(value, args, newTarget === proxy ? value : newTarget);
+        },
+        apply: (_t, thisArg, args) => Reflect.apply(load(), thisArg, args),
+    });
+    return proxy;
+}

package/dist/node-wrapper.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Package-manager processes in actor containers.
+ *
+ * The base images preload the SDK's browser patch into every Node process
+ * (NODE_OPTIONS=--import /opt/scrapely/register.mjs). `npm start` is two Node
+ * processes: npm, which only spawns the start script, and the actor. npm itself
+ * costs ~0.6 s at 1 core and ~2.7 s at 0.25 core (1 GB run) under gVisor.
+ *
+ * - npm / npx / yarn / pnpm processes never run actor code, so they get no
+ *   loader hooks. Their children inherit NODE_OPTIONS and get the preload.
+ * - `npm start` / `npm run <script>` whose script is a plain `node <args>`
+ *   (no shell syntax, no pre/post scripts, no .npmrc) runs that node process
+ *   directly instead of loading npm: the preload spawns it with the
+ *   environment npm would give it, forwards signals and exits with its code.
+ *   Set SCRAPELY_NPM_START_SHORTCUT=0 to always use npm.
+ *
+ * Loaded in every Node process by the preload: keep it free of heavy imports.
+ */
+import type { ChildProcess } from 'node:child_process';
+type PackageManager = 'npm' | 'npx' | 'yarn' | 'pnpm';
+/** @internal Which package manager this process is running, from its main script. */
+export declare function detectPackageManager(mainScript?: string | undefined): PackageManager | undefined;
+export interface NpmStartPlan {
+    /** Node binary and its arguments, as the start script would run them. */
+    command: string;
+    args: string[];
+    env: NodeJS.ProcessEnv;
+    cwd: string;
+    scriptName: string;
+}
+/**
+ * Plan for running `npm start` / `npm run <script>` without npm, or undefined
+ * when npm must run it (anything not provably a plain node invocation).
+ *
+ * @internal Exported for tests.
+ */
+export declare function planNpmStart(npmArgs?: string[], options?: {
+    cwd?: string;
+    env?: NodeJS.ProcessEnv;
+    execPath?: string;
+    npmCliPath?: string;
+}): NpmStartPlan | undefined;
+/**
+ * Run the planned node process in place of npm: inherit stdio, forward
+ * signals, exit with its exit code (128 + signal number when it was killed).
+ *
+ * @internal Exported for tests.
+ */
+export declare function runNpmStartPlan(plan: NpmStartPlan): ChildProcess;
+/**
+ * Called by the preload in every Node process. Returns true for
+ * package-manager processes (no loader hooks needed). For a shortcut-able
+ * `npm start`, the main module load (npm-cli.js) is intercepted: the start
+ * script's node process runs instead and npm itself is never loaded. If Node
+ * ever loaded the main module another way, npm simply runs as before.
+ */
+export declare function handlePackageManagerProcess(): boolean;
+export {};

package/dist/node-wrapper.js

@@ -0,0 +1,270 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectPackageManager = detectPackageManager;
+exports.planNpmStart = planNpmStart;
+exports.runNpmStartPlan = runNpmStartPlan;
+exports.handlePackageManagerProcess = handlePackageManagerProcess;
+/** Main-script basenames of package-manager CLIs (npm's bin, corepack's shims). */
+const PACKAGE_MANAGER_SCRIPTS = {
+    'npm-cli.js': 'npm',
+    'npx-cli.js': 'npx',
+    'yarn.js': 'yarn',
+    'yarn.cjs': 'yarn',
+    'pnpm.cjs': 'pnpm',
+    'pnpm.js': 'pnpm',
+    'pnpm.mjs': 'pnpm',
+};
+/** @internal Which package manager this process is running, from its main script. */
+function detectPackageManager(mainScript = process.argv[1]) {
+    if (!mainScript)
+        return undefined;
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const path = require('path');
+    let resolved = mainScript;
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        resolved = require('fs').realpathSync(mainScript);
+    }
+    catch {
+        // keep the path as given
+    }
+    const base = path.basename(resolved);
+    const kind = PACKAGE_MANAGER_SCRIPTS[base];
+    if (!kind)
+        return undefined;
+    // yarn.js / pnpm.cjs are generic names: require a package-manager install path.
+    if (kind === 'npm' || kind === 'npx')
+        return /[/\\]npm[/\\]bin[/\\]/.test(resolved) ? kind : undefined;
+    return /[/\\](yarn|pnpm|corepack|\.yarn)[/\\]|[/\\]yarn-[^/\\]+[/\\]|[/\\]\.pnpm[/\\]/.test(resolved) ? kind : undefined;
+}
+/** A script token that needs no shell: no quotes, expansions, globs, operators or redirects. */
+const PLAIN_TOKEN_RE = /^[A-Za-z0-9_@%+=:,./-]+$/;
+/** npm flags accepted before/after the script name; anything else falls back to npm. */
+const QUIET_FLAGS = new Set(['--silent', '-s', '--quiet', '-q', '--loglevel=silent', '--loglevel=error', '--loglevel=warn']);
+/** npm config keys that change how (or whether) a script runs: never shortcut when set. */
+const SCRIPT_CONFIG_KEYS = [
+    'node-options',
+    'script-shell',
+    'ignore-scripts',
+    'scripts-prepend-node-path',
+    'workspace',
+    'workspaces',
+    'include-workspace-root',
+    'userconfig',
+    'globalconfig',
+];
+const SCRIPT_CONFIG_ENV_RE = new RegExp(`^npm_config_(${[...SCRIPT_CONFIG_KEYS, 'prefix'].map((k) => k.replace(/-/g, '_')).join('|')})$`, 'i');
+/**
+ * Whether an npmrc file sets any of SCRIPT_CONFIG_KEYS. Other settings
+ * (progress=false, loglevel, registry, auth, ...) don't affect running a
+ * script — Apify's Dockerfiles write `progress=false` to ~/.npmrc.
+ * Unreadable or unparseable files count as setting them.
+ */
+function readNpmrc(file) {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const fs = require('fs');
+    let text;
+    try {
+        if (!fs.existsSync(file))
+            return { affectsScripts: false };
+        text = fs.readFileSync(file, 'utf8');
+    }
+    catch {
+        return { affectsScripts: true };
+    }
+    let prefix;
+    for (const raw of text.split(/\r?\n/)) {
+        const line = raw.trim();
+        if (!line || line.startsWith('#') || line.startsWith(';'))
+            continue;
+        if (line.startsWith('['))
+            return { affectsScripts: true }; // ini sections: unexpected, be safe
+        const eq = line.indexOf('=');
+        const key = (eq === -1 ? line : line.slice(0, eq)).trim().toLowerCase().replace(/_/g, '-');
+        if (SCRIPT_CONFIG_KEYS.includes(key))
+            return { affectsScripts: true };
+        // `prefix` only moves the global npmrc ($prefix/etc/npmrc), e.g. Homebrew's builtin npmrc.
+        if (key === 'prefix' && eq !== -1)
+            prefix = line.slice(eq + 1).trim();
+    }
+    return { affectsScripts: false, prefix };
+}
+/**
+ * Plan for running `npm start` / `npm run <script>` without npm, or undefined
+ * when npm must run it (anything not provably a plain node invocation).
+ *
+ * @internal Exported for tests.
+ */
+function planNpmStart(npmArgs = process.argv.slice(2), options = {}) {
+    const env = options.env ?? process.env;
+    if (env.SCRAPELY_NPM_START_SHORTCUT === '0')
+        return undefined;
+    if (Object.keys(env).some((k) => SCRIPT_CONFIG_ENV_RE.test(k)))
+        return undefined;
+    // `start [flags] [-- extra]` or `run|run-script <name> [flags] [-- extra]`
+    const dashDash = npmArgs.indexOf('--');
+    const head = dashDash === -1 ? npmArgs : npmArgs.slice(0, dashDash);
+    const extra = dashDash === -1 ? [] : npmArgs.slice(dashDash + 1);
+    const positional = head.filter((a) => !a.startsWith('-'));
+    if (head.some((a) => a.startsWith('-') && !QUIET_FLAGS.has(a)))
+        return undefined;
+    let scriptName;
+    if (positional.length === 1 && positional[0] === 'start') {
+        scriptName = 'start';
+    }
+    else if (positional.length === 2 && (positional[0] === 'run' || positional[0] === 'run-script')) {
+        scriptName = positional[1];
+    }
+    else {
+        return undefined;
+    }
+    if (!extra.every((a) => PLAIN_TOKEN_RE.test(a)))
+        return undefined;
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const fs = require('fs');
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const path = require('path');
+    const cwd = options.cwd ?? process.cwd();
+    const packageJsonPath = path.join(cwd, 'package.json');
+    const execPath = options.execPath ?? process.execPath;
+    const npmCliPath = options.npmCliPath ?? process.argv[1];
+    // The npmrc files npm reads: builtin (npm's install dir), global
+    // ($prefix/etc/npmrc; prefix defaults to node's install dir unless an
+    // npmrc moves it), user, project.
+    const home = env.HOME || require('os').homedir();
+    const builtin = npmCliPath ? readNpmrc(path.join(path.dirname(path.dirname(npmCliPath)), 'npmrc')) : { affectsScripts: false };
+    const user = home ? readNpmrc(path.join(home, '.npmrc')) : { affectsScripts: false };
+    const project = readNpmrc(path.join(cwd, '.npmrc'));
+    const prefix = project.prefix ?? user.prefix ?? builtin.prefix ?? path.dirname(path.dirname(execPath));
+    const global = readNpmrc(path.join(prefix, 'etc', 'npmrc'));
+    if ([builtin, global, user, project].some((rc) => rc.affectsScripts))
+        return undefined;
+    let pkg;
+    try {
+        pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+    if (!pkg || typeof pkg !== 'object' || pkg.workspaces)
+        return undefined;
+    const scripts = pkg.scripts ?? {};
+    const script = scripts[scriptName];
+    if (typeof script !== 'string')
+        return undefined;
+    if (scripts[`pre${scriptName}`] !== undefined || scripts[`post${scriptName}`] !== undefined)
+        return undefined;
+    const tokens = script.trim().split(/[ \t]+/);
+    if (tokens[0] !== 'node' || tokens.length < 2)
+        return undefined;
+    if (!tokens.every((t) => PLAIN_TOKEN_RE.test(t)))
+        return undefined;
+    const silent = head.some((a) => QUIET_FLAGS.has(a));
+    // The script environment npm provides (@npmcli/run-script): local bins on
+    // PATH, package metadata, lifecycle names.
+    const binDirs = [];
+    for (let dir = cwd;;) {
+        binDirs.push(path.join(dir, 'node_modules', '.bin'));
+        const parent = path.dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    // A local `node` binary would win on npm's PATH.
+    if (binDirs.some((dir) => fs.existsSync(path.join(dir, 'node'))))
+        return undefined;
+    const childEnv = {
+        ...env,
+        PATH: [...binDirs, env.PATH].filter(Boolean).join(path.delimiter),
+        INIT_CWD: env.INIT_CWD ?? cwd,
+        NODE: execPath,
+        npm_node_execpath: execPath,
+        npm_lifecycle_event: scriptName,
+        npm_lifecycle_script: script,
+        npm_package_json: packageJsonPath,
+        npm_command: scriptName === 'start' ? 'start' : 'run-script',
+    };
+    if (npmCliPath)
+        childEnv.npm_execpath = npmCliPath;
+    if (typeof pkg.name === 'string')
+        childEnv.npm_package_name = pkg.name;
+    if (typeof pkg.version === 'string')
+        childEnv.npm_package_version = pkg.version;
+    if (silent)
+        childEnv.npm_config_loglevel = 'silent';
+    return { command: execPath, args: [...tokens.slice(1), ...extra], env: childEnv, cwd, scriptName };
+}
+const FORWARDED_SIGNALS = ['SIGTERM', 'SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR2'];
+/**
+ * Run the planned node process in place of npm: inherit stdio, forward
+ * signals, exit with its exit code (128 + signal number when it was killed).
+ *
+ * @internal Exported for tests.
+ */
+function runNpmStartPlan(plan) {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const { spawn } = require('child_process');
+    const child = spawn(plan.command, plan.args, { cwd: plan.cwd, env: plan.env, stdio: 'inherit' });
+    const forward = (signal) => {
+        try {
+            child.kill(signal);
+        }
+        catch {
+            // already exited
+        }
+    };
+    for (const signal of FORWARDED_SIGNALS)
+        process.on(signal, forward);
+    child.on('error', (err) => {
+        console.error(`[scrapely] could not start '${plan.command} ${plan.args.join(' ')}': ${err.message}`);
+        process.exit(1);
+    });
+    child.on('exit', (code, signal) => {
+        for (const s of FORWARDED_SIGNALS)
+            process.removeListener(s, forward);
+        if (code !== null)
+            process.exit(code);
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const signals = require('os').constants.signals;
+        process.exit(128 + (signal ? signals[signal] ?? 1 : 1));
+    });
+    return child;
+}
+/**
+ * Called by the preload in every Node process. Returns true for
+ * package-manager processes (no loader hooks needed). For a shortcut-able
+ * `npm start`, the main module load (npm-cli.js) is intercepted: the start
+ * script's node process runs instead and npm itself is never loaded. If Node
+ * ever loaded the main module another way, npm simply runs as before.
+ */
+function handlePackageManagerProcess() {
+    const manager = detectPackageManager();
+    if (!manager)
+        return false;
+    if (manager !== 'npm')
+        return true;
+    let plan;
+    try {
+        plan = planNpmStart();
+    }
+    catch {
+        plan = undefined;
+    }
+    if (!plan)
+        return true;
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const Module = require('module');
+    const origLoad = Module._load;
+    if (typeof origLoad !== 'function')
+        return true;
+    const planned = plan;
+    Module._load = function (request, parent, isMain) {
+        if (isMain) {
+            Module._load = origLoad;
+            runNpmStartPlan(planned);
+            return {};
+        }
+        return origLoad.apply(this, [request, parent, isMain]);
+    };
+    return true;
+}

package/dist/patch-browser-pool.d.ts

@@ -59,8 +59,48 @@ export declare function patchCrawleeCrawlerModule(mod: any): void;
  * @internal Exported for tests.
  */
 export declare function patchLoadedCrawleeCopies(): void;
+/**
+ * Patch the launcher classes exported by one loaded Crawlee launcher file (or
+ * package index). Idempotent.
+ *
+ * - `cdpUrl` becomes a valid launcher option (PlaywrightLauncher and
+ *   PuppeteerLauncher copy BrowserLauncher's shape when their file loads, so
+ *   each class gets it; browser-launcher.js loads first, so the copies inherit it).
+ * - Local launches get gVisor-safe Chrome args under runsc. Playwright/Puppeteer
+ *   pool launches use CDP and skip this.
+ *
+ * @internal Exported for tests.
+ */
+export declare function patchBrowserLauncherModule(mod: any, parentFilename?: string): void;
 /** @internal Exported for tests. */
 export declare function patchRawBrowserType(chromium: {
     connectOverCDP: (url: string, opts?: unknown) => Promise<any>;
 }, browserType: any, name: string): void;
+/**
+ * Patch puppeteer's PuppeteerNode class (puppeteer-core `node/PuppeteerNode.js`,
+ * shared by puppeteer and puppeteer-core, CJS and ESM builds). The package
+ * entry creates its default instance right after this class loads, and the
+ * constructor binds `launch`/`connect` from the prototype, so patching the
+ * prototype first also covers the named exports (`import { launch } from
+ * 'puppeteer'`), which are those bound methods. Idempotent per class.
+ *
+ * @internal Exported for tests.
+ */
+export declare function patchPuppeteerNodeClass(PuppeteerNode: any): void;
+/**
+ * Patch copies that loaded before this module (e.g. a local run that imports
+ * crawlee or playwright before scrapely). Copies loaded later are patched by
+ * the loader hooks.
+ *
+ * @internal Exported for tests.
+ */
+export declare function patchLoadedModules(): void;
+/** @internal Global the ESM load hook's appended code calls. */
+export declare const PUPPETEER_NODE_PATCH_GLOBAL = "scrapely.patchPuppeteerNodeClass";
+/** puppeteer-core's ESM PuppeteerNode.js (URL): lib/esm/puppeteer/... up to v24, lib/puppeteer/... (ESM only) from v25. */
+export declare const PUPPETEER_NODE_ESM_URL_RE: RegExp;
+/** Appended to the ESM PuppeteerNode.js source; runs right after the class is defined. */
+export declare const PUPPETEER_NODE_ESM_SUFFIX: string;
+/** @internal Whether an ESM import of puppeteer / puppeteer-core could succeed in this process. */
+export declare function esmPuppeteerInstalled(): boolean;
 export {};