scrapely 1.2.25 → 1.2.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +14 -3
- package/dist/actor.js +63 -22
- package/dist/charging.d.ts +7 -2
- package/dist/charging.js +16 -4
- package/dist/charging_client.d.ts +1 -3
- package/dist/charging_client.js +3 -5
- package/dist/client-loader.d.ts +20 -0
- package/dist/client-loader.js +42 -0
- package/dist/esm-loader-hooks.d.mts +20 -0
- package/dist/esm-loader-hooks.mjs +21 -0
- package/dist/index.d.mts +18 -0
- package/dist/index.d.ts +33 -12
- package/dist/index.js +7 -29
- package/dist/index.mjs +39 -0
- package/dist/lazy-exports.d.ts +14 -0
- package/dist/lazy-exports.js +132 -0
- package/dist/node-wrapper.d.ts +58 -0
- package/dist/node-wrapper.js +270 -0
- package/dist/patch-browser-pool.d.ts +40 -0
- package/dist/patch-browser-pool.js +349 -114
- package/dist/proxy-configuration.js +1 -1
- package/package.json +16 -2
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ESM entry (`import ... from 'scrapely'`, and the `apify` shim's
|
|
3
|
+
* `export * from 'scrapely'`).
|
|
4
|
+
*
|
|
5
|
+
* Re-exports the CommonJS entry, plus ESM stand-ins for the names that load
|
|
6
|
+
* heavy packages (Crawlee crawlers, scrapely-client): Node reads every
|
|
7
|
+
* detected export of a CommonJS module when ESM imports it, so index.js hides
|
|
8
|
+
* those names from cjs-module-lexer and they are exported here instead. They
|
|
9
|
+
* load on first use (see lazy-exports.ts).
|
|
10
|
+
*/
|
|
11
|
+
import { createRequire } from 'node:module';
|
|
12
|
+
const require = createRequire(import.meta.url);
|
|
13
|
+
const sdk = require('./index.js');
|
|
14
|
+
const { lazyExport } = require('./lazy-exports.js');
|
|
15
|
+
const { onScrapelyClientLoaded } = require('./client-loader.js');
|
|
16
|
+
export * from './index.js';
|
|
17
|
+
/** `import scrapely from 'scrapely'`: the CommonJS exports object, as before this entry existed. */
|
|
18
|
+
export default sdk;
|
|
19
|
+
export const PlaywrightCrawler = lazyExport('PlaywrightCrawler', 'class');
|
|
20
|
+
export const PuppeteerCrawler = lazyExport('PuppeteerCrawler', 'class');
|
|
21
|
+
export const CheerioCrawler = lazyExport('CheerioCrawler', 'class');
|
|
22
|
+
export const HttpCrawler = lazyExport('HttpCrawler', 'class');
|
|
23
|
+
export const BasicCrawler = lazyExport('BasicCrawler', 'class');
|
|
24
|
+
export const BrowserCrawler = lazyExport('BrowserCrawler', 'class');
|
|
25
|
+
export const JSDOMCrawler = lazyExport('JSDOMCrawler', 'class');
|
|
26
|
+
export const LinkeDOMCrawler = lazyExport('LinkeDOMCrawler', 'class');
|
|
27
|
+
export const ApifyClient = lazyExport('ApifyClient', 'class');
|
|
28
|
+
export const playwrightUtils = lazyExport('playwrightUtils', 'object');
|
|
29
|
+
export const puppeteerUtils = lazyExport('puppeteerUtils', 'object');
|
|
30
|
+
// The client's AsyncLocalStorage: methods (run, getStore, ...) act on the real instance.
|
|
31
|
+
export const pushDataChargingContext = lazyExport('pushDataChargingContext', 'object');
|
|
32
|
+
// A symbol cannot be stood in for: a live binding, set as soon as the SDK
|
|
33
|
+
// loads scrapely-client (an Actor creating its API client, charging,
|
|
34
|
+
// ApifyClient above, ...). Only clients from that module carry the marker,
|
|
35
|
+
// so it is always set before any client could be checked for it.
|
|
36
|
+
export let USES_PUSH_DATA_INTERCEPTION;
|
|
37
|
+
onScrapelyClientLoaded((client) => {
|
|
38
|
+
USES_PUSH_DATA_INTERCEPTION = client.USES_PUSH_DATA_INTERCEPTION;
|
|
39
|
+
});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** name → [package, export]. */
|
|
2
|
+
export declare const LAZY_EXPORTS: Readonly<Record<string, readonly [string, string]>>;
|
|
3
|
+
/** Resolve one lazy export (loads its package on first call). */
|
|
4
|
+
export declare function loadLazyExport(name: string): unknown;
|
|
5
|
+
/** Define every lazy export as a self-replacing getter on a CommonJS exports object. */
|
|
6
|
+
export declare function defineLazyExports(target: Record<string, unknown>): void;
|
|
7
|
+
/**
|
|
8
|
+
* ESM stand-in for a lazy export: a Proxy that loads the real value on first
|
|
9
|
+
* use and forwards everything to it. For classes it is constructible and
|
|
10
|
+
* extendable (`new X()`, `class Y extends X`, `instanceof X`, statics); for
|
|
11
|
+
* objects (playwrightUtils, ...) property access works as usual. The one
|
|
12
|
+
* visible difference: the stand-in is not `===` the package's own export.
|
|
13
|
+
*/
|
|
14
|
+
export declare function lazyExport<T>(name: string, kind: 'class' | 'object'): T;
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LAZY_EXPORTS = void 0;
|
|
4
|
+
exports.loadLazyExport = loadLazyExport;
|
|
5
|
+
exports.defineLazyExports = defineLazyExports;
|
|
6
|
+
exports.lazyExport = lazyExport;
|
|
7
|
+
/**
|
|
8
|
+
* Public SDK names that load heavy packages: Crawlee's crawlers (jsdom,
|
|
9
|
+
* linkedom, cheerio, the browser pool, playwright) and scrapely-client.
|
|
10
|
+
* Importing the SDK used to load all of them, ~4 s at 1 core under gVisor;
|
|
11
|
+
* they now load on first access.
|
|
12
|
+
*
|
|
13
|
+
* - CommonJS (`require('scrapely').PlaywrightCrawler`): a getter on the
|
|
14
|
+
* package exports that requires the package once, then becomes a plain value.
|
|
15
|
+
* - ESM (`import { PlaywrightCrawler } from 'scrapely'` / 'apify'): index.mts
|
|
16
|
+
* exports a lazy stand-in per name (see lazyExport). Node reads every
|
|
17
|
+
* detected export of a CommonJS module when ESM imports it, so these getters
|
|
18
|
+
* are deliberately defined in a form cjs-module-lexer does not detect.
|
|
19
|
+
*
|
|
20
|
+
* Loaded by the SDK entry: keep this file free of heavy imports.
|
|
21
|
+
*/
|
|
22
|
+
const client_loader_js_1 = require("./client-loader.js");
|
|
23
|
+
/** name → [package, export]. */
|
|
24
|
+
exports.LAZY_EXPORTS = {
|
|
25
|
+
PlaywrightCrawler: ['@crawlee/playwright', 'PlaywrightCrawler'],
|
|
26
|
+
PuppeteerCrawler: ['@crawlee/puppeteer', 'PuppeteerCrawler'],
|
|
27
|
+
CheerioCrawler: ['@crawlee/cheerio', 'CheerioCrawler'],
|
|
28
|
+
HttpCrawler: ['@crawlee/http', 'HttpCrawler'],
|
|
29
|
+
BasicCrawler: ['@crawlee/basic', 'BasicCrawler'],
|
|
30
|
+
BrowserCrawler: ['@crawlee/browser', 'BrowserCrawler'],
|
|
31
|
+
JSDOMCrawler: ['@crawlee/jsdom', 'JSDOMCrawler'],
|
|
32
|
+
LinkeDOMCrawler: ['@crawlee/linkedom', 'LinkeDOMCrawler'],
|
|
33
|
+
playwrightUtils: ['@crawlee/playwright', 'playwrightUtils'],
|
|
34
|
+
puppeteerUtils: ['@crawlee/puppeteer', 'puppeteerUtils'],
|
|
35
|
+
ApifyClient: ['scrapely-client', 'ApifyClient'],
|
|
36
|
+
USES_PUSH_DATA_INTERCEPTION: ['scrapely-client', 'USES_PUSH_DATA_INTERCEPTION'],
|
|
37
|
+
pushDataChargingContext: ['scrapely-client', 'pushDataChargingContext'],
|
|
38
|
+
};
|
|
39
|
+
/** Resolve one lazy export (loads its package on first call). */
|
|
40
|
+
function loadLazyExport(name) {
|
|
41
|
+
const entry = exports.LAZY_EXPORTS[name];
|
|
42
|
+
if (!entry)
|
|
43
|
+
throw new Error(`Unknown lazy export: ${name}`);
|
|
44
|
+
if (entry[0] === 'scrapely-client')
|
|
45
|
+
return (0, client_loader_js_1.scrapelyClient)()[entry[1]];
|
|
46
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
47
|
+
return require(entry[0])[entry[1]];
|
|
48
|
+
}
|
|
49
|
+
/** Define every lazy export as a self-replacing getter on a CommonJS exports object. */
|
|
50
|
+
function defineLazyExports(target) {
|
|
51
|
+
for (const name of Object.keys(exports.LAZY_EXPORTS)) {
|
|
52
|
+
Object.defineProperty(target, name, {
|
|
53
|
+
enumerable: true,
|
|
54
|
+
configurable: true,
|
|
55
|
+
get() {
|
|
56
|
+
const value = loadLazyExport(name);
|
|
57
|
+
Object.defineProperty(target, name, { value, enumerable: true, configurable: true, writable: false });
|
|
58
|
+
return value;
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* ESM stand-in for a lazy export: a Proxy that loads the real value on first
|
|
65
|
+
* use and forwards everything to it. For classes it is constructible and
|
|
66
|
+
* extendable (`new X()`, `class Y extends X`, `instanceof X`, statics); for
|
|
67
|
+
* objects (playwrightUtils, ...) property access works as usual. The one
|
|
68
|
+
* visible difference: the stand-in is not `===` the package's own export.
|
|
69
|
+
*/
|
|
70
|
+
function lazyExport(name, kind) {
|
|
71
|
+
let loaded = false;
|
|
72
|
+
let real;
|
|
73
|
+
const load = () => {
|
|
74
|
+
if (!loaded) {
|
|
75
|
+
real = loadLazyExport(name);
|
|
76
|
+
loaded = true;
|
|
77
|
+
}
|
|
78
|
+
return real;
|
|
79
|
+
};
|
|
80
|
+
const boundMethods = new Map();
|
|
81
|
+
const bound = (fn, self) => {
|
|
82
|
+
let b = boundMethods.get(fn);
|
|
83
|
+
if (!b) {
|
|
84
|
+
b = fn.bind(self);
|
|
85
|
+
boundMethods.set(fn, b);
|
|
86
|
+
}
|
|
87
|
+
return b;
|
|
88
|
+
};
|
|
89
|
+
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
|
90
|
+
const target = kind === 'class' ? function LazyExport() { } : {};
|
|
91
|
+
const proxy = new Proxy(target, {
|
|
92
|
+
get(_t, prop, receiver) {
|
|
93
|
+
const value = load();
|
|
94
|
+
const result = Reflect.get(value, prop, receiver === proxy ? value : receiver);
|
|
95
|
+
// Object stand-ins: methods run on the real object (private fields,
|
|
96
|
+
// internal slots), e.g. pushDataChargingContext.run().
|
|
97
|
+
if (kind === 'object' && typeof result === 'function')
|
|
98
|
+
return bound(result, value);
|
|
99
|
+
return result;
|
|
100
|
+
},
|
|
101
|
+
set(_t, prop, newValue, receiver) {
|
|
102
|
+
const value = load();
|
|
103
|
+
return Reflect.set(value, prop, newValue, receiver === proxy ? value : receiver);
|
|
104
|
+
},
|
|
105
|
+
has: (_t, prop) => Reflect.has(load(), prop),
|
|
106
|
+
ownKeys: () => {
|
|
107
|
+
const keys = Reflect.ownKeys(load());
|
|
108
|
+
if (kind === 'class' && !keys.includes('prototype'))
|
|
109
|
+
keys.push('prototype');
|
|
110
|
+
return keys;
|
|
111
|
+
},
|
|
112
|
+
getOwnPropertyDescriptor(_t, prop) {
|
|
113
|
+
const desc = Reflect.getOwnPropertyDescriptor(load(), prop);
|
|
114
|
+
if (!desc)
|
|
115
|
+
return undefined;
|
|
116
|
+
// Stay compatible with the stand-in target (Proxy invariants): only
|
|
117
|
+
// the target function's own `prototype` may be non-configurable.
|
|
118
|
+
if (kind === 'class' && prop === 'prototype')
|
|
119
|
+
return { ...desc, writable: true, configurable: false };
|
|
120
|
+
return { ...desc, configurable: true };
|
|
121
|
+
},
|
|
122
|
+
defineProperty: (_t, prop, desc) => Reflect.defineProperty(load(), prop, desc),
|
|
123
|
+
deleteProperty: (_t, prop) => Reflect.deleteProperty(load(), prop),
|
|
124
|
+
getPrototypeOf: () => Reflect.getPrototypeOf(load()),
|
|
125
|
+
construct(_t, args, newTarget) {
|
|
126
|
+
const value = load();
|
|
127
|
+
return Reflect.construct(value, args, newTarget === proxy ? value : newTarget);
|
|
128
|
+
},
|
|
129
|
+
apply: (_t, thisArg, args) => Reflect.apply(load(), thisArg, args),
|
|
130
|
+
});
|
|
131
|
+
return proxy;
|
|
132
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Package-manager processes in actor containers.
|
|
3
|
+
*
|
|
4
|
+
* The base images preload the SDK's browser patch into every Node process
|
|
5
|
+
* (NODE_OPTIONS=--import /opt/scrapely/register.mjs). `npm start` is two Node
|
|
6
|
+
* processes: npm, which only spawns the start script, and the actor. npm itself
|
|
7
|
+
* costs ~0.6 s at 1 core and ~2.7 s at 0.25 core (1 GB run) under gVisor.
|
|
8
|
+
*
|
|
9
|
+
* - npm / npx / yarn / pnpm processes never run actor code, so they get no
|
|
10
|
+
* loader hooks. Their children inherit NODE_OPTIONS and get the preload.
|
|
11
|
+
* - `npm start` / `npm run <script>` whose script is a plain `node <args>`
|
|
12
|
+
* (no shell syntax, no pre/post scripts, no .npmrc) runs that node process
|
|
13
|
+
* directly instead of loading npm: the preload spawns it with the
|
|
14
|
+
* environment npm would give it, forwards signals and exits with its code.
|
|
15
|
+
* Set SCRAPELY_NPM_START_SHORTCUT=0 to always use npm.
|
|
16
|
+
*
|
|
17
|
+
* Loaded in every Node process by the preload: keep it free of heavy imports.
|
|
18
|
+
*/
|
|
19
|
+
import type { ChildProcess } from 'node:child_process';
|
|
20
|
+
type PackageManager = 'npm' | 'npx' | 'yarn' | 'pnpm';
|
|
21
|
+
/** @internal Which package manager this process is running, from its main script. */
|
|
22
|
+
export declare function detectPackageManager(mainScript?: string | undefined): PackageManager | undefined;
|
|
23
|
+
export interface NpmStartPlan {
|
|
24
|
+
/** Node binary and its arguments, as the start script would run them. */
|
|
25
|
+
command: string;
|
|
26
|
+
args: string[];
|
|
27
|
+
env: NodeJS.ProcessEnv;
|
|
28
|
+
cwd: string;
|
|
29
|
+
scriptName: string;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Plan for running `npm start` / `npm run <script>` without npm, or undefined
|
|
33
|
+
* when npm must run it (anything not provably a plain node invocation).
|
|
34
|
+
*
|
|
35
|
+
* @internal Exported for tests.
|
|
36
|
+
*/
|
|
37
|
+
export declare function planNpmStart(npmArgs?: string[], options?: {
|
|
38
|
+
cwd?: string;
|
|
39
|
+
env?: NodeJS.ProcessEnv;
|
|
40
|
+
execPath?: string;
|
|
41
|
+
npmCliPath?: string;
|
|
42
|
+
}): NpmStartPlan | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* Run the planned node process in place of npm: inherit stdio, forward
|
|
45
|
+
* signals, exit with its exit code (128 + signal number when it was killed).
|
|
46
|
+
*
|
|
47
|
+
* @internal Exported for tests.
|
|
48
|
+
*/
|
|
49
|
+
export declare function runNpmStartPlan(plan: NpmStartPlan): ChildProcess;
|
|
50
|
+
/**
|
|
51
|
+
* Called by the preload in every Node process. Returns true for
|
|
52
|
+
* package-manager processes (no loader hooks needed). For a shortcut-able
|
|
53
|
+
* `npm start`, the main module load (npm-cli.js) is intercepted: the start
|
|
54
|
+
* script's node process runs instead and npm itself is never loaded. If Node
|
|
55
|
+
* ever loaded the main module another way, npm simply runs as before.
|
|
56
|
+
*/
|
|
57
|
+
export declare function handlePackageManagerProcess(): boolean;
|
|
58
|
+
export {};
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.detectPackageManager = detectPackageManager;
|
|
4
|
+
exports.planNpmStart = planNpmStart;
|
|
5
|
+
exports.runNpmStartPlan = runNpmStartPlan;
|
|
6
|
+
exports.handlePackageManagerProcess = handlePackageManagerProcess;
|
|
7
|
+
/** Main-script basenames of package-manager CLIs (npm's bin, corepack's shims). */
|
|
8
|
+
const PACKAGE_MANAGER_SCRIPTS = {
|
|
9
|
+
'npm-cli.js': 'npm',
|
|
10
|
+
'npx-cli.js': 'npx',
|
|
11
|
+
'yarn.js': 'yarn',
|
|
12
|
+
'yarn.cjs': 'yarn',
|
|
13
|
+
'pnpm.cjs': 'pnpm',
|
|
14
|
+
'pnpm.js': 'pnpm',
|
|
15
|
+
'pnpm.mjs': 'pnpm',
|
|
16
|
+
};
|
|
17
|
+
/** @internal Which package manager this process is running, from its main script. */
|
|
18
|
+
function detectPackageManager(mainScript = process.argv[1]) {
|
|
19
|
+
if (!mainScript)
|
|
20
|
+
return undefined;
|
|
21
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
22
|
+
const path = require('path');
|
|
23
|
+
let resolved = mainScript;
|
|
24
|
+
try {
|
|
25
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
26
|
+
resolved = require('fs').realpathSync(mainScript);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// keep the path as given
|
|
30
|
+
}
|
|
31
|
+
const base = path.basename(resolved);
|
|
32
|
+
const kind = PACKAGE_MANAGER_SCRIPTS[base];
|
|
33
|
+
if (!kind)
|
|
34
|
+
return undefined;
|
|
35
|
+
// yarn.js / pnpm.cjs are generic names: require a package-manager install path.
|
|
36
|
+
if (kind === 'npm' || kind === 'npx')
|
|
37
|
+
return /[/\\]npm[/\\]bin[/\\]/.test(resolved) ? kind : undefined;
|
|
38
|
+
return /[/\\](yarn|pnpm|corepack|\.yarn)[/\\]|[/\\]yarn-[^/\\]+[/\\]|[/\\]\.pnpm[/\\]/.test(resolved) ? kind : undefined;
|
|
39
|
+
}
|
|
40
|
+
/** A script token that needs no shell: no quotes, expansions, globs, operators or redirects. */
|
|
41
|
+
const PLAIN_TOKEN_RE = /^[A-Za-z0-9_@%+=:,./-]+$/;
|
|
42
|
+
/** npm flags accepted before/after the script name; anything else falls back to npm. */
|
|
43
|
+
const QUIET_FLAGS = new Set(['--silent', '-s', '--quiet', '-q', '--loglevel=silent', '--loglevel=error', '--loglevel=warn']);
|
|
44
|
+
/** npm config keys that change how (or whether) a script runs: never shortcut when set. */
|
|
45
|
+
const SCRIPT_CONFIG_KEYS = [
|
|
46
|
+
'node-options',
|
|
47
|
+
'script-shell',
|
|
48
|
+
'ignore-scripts',
|
|
49
|
+
'scripts-prepend-node-path',
|
|
50
|
+
'workspace',
|
|
51
|
+
'workspaces',
|
|
52
|
+
'include-workspace-root',
|
|
53
|
+
'userconfig',
|
|
54
|
+
'globalconfig',
|
|
55
|
+
];
|
|
56
|
+
const SCRIPT_CONFIG_ENV_RE = new RegExp(`^npm_config_(${[...SCRIPT_CONFIG_KEYS, 'prefix'].map((k) => k.replace(/-/g, '_')).join('|')})$`, 'i');
|
|
57
|
+
/**
|
|
58
|
+
* Whether an npmrc file sets any of SCRIPT_CONFIG_KEYS. Other settings
|
|
59
|
+
* (progress=false, loglevel, registry, auth, ...) don't affect running a
|
|
60
|
+
* script — Apify's Dockerfiles write `progress=false` to ~/.npmrc.
|
|
61
|
+
* Unreadable or unparseable files count as setting them.
|
|
62
|
+
*/
|
|
63
|
+
function readNpmrc(file) {
|
|
64
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
65
|
+
const fs = require('fs');
|
|
66
|
+
let text;
|
|
67
|
+
try {
|
|
68
|
+
if (!fs.existsSync(file))
|
|
69
|
+
return { affectsScripts: false };
|
|
70
|
+
text = fs.readFileSync(file, 'utf8');
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return { affectsScripts: true };
|
|
74
|
+
}
|
|
75
|
+
let prefix;
|
|
76
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
77
|
+
const line = raw.trim();
|
|
78
|
+
if (!line || line.startsWith('#') || line.startsWith(';'))
|
|
79
|
+
continue;
|
|
80
|
+
if (line.startsWith('['))
|
|
81
|
+
return { affectsScripts: true }; // ini sections: unexpected, be safe
|
|
82
|
+
const eq = line.indexOf('=');
|
|
83
|
+
const key = (eq === -1 ? line : line.slice(0, eq)).trim().toLowerCase().replace(/_/g, '-');
|
|
84
|
+
if (SCRIPT_CONFIG_KEYS.includes(key))
|
|
85
|
+
return { affectsScripts: true };
|
|
86
|
+
// `prefix` only moves the global npmrc ($prefix/etc/npmrc), e.g. Homebrew's builtin npmrc.
|
|
87
|
+
if (key === 'prefix' && eq !== -1)
|
|
88
|
+
prefix = line.slice(eq + 1).trim();
|
|
89
|
+
}
|
|
90
|
+
return { affectsScripts: false, prefix };
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Plan for running `npm start` / `npm run <script>` without npm, or undefined
|
|
94
|
+
* when npm must run it (anything not provably a plain node invocation).
|
|
95
|
+
*
|
|
96
|
+
* @internal Exported for tests.
|
|
97
|
+
*/
|
|
98
|
+
function planNpmStart(npmArgs = process.argv.slice(2), options = {}) {
|
|
99
|
+
const env = options.env ?? process.env;
|
|
100
|
+
if (env.SCRAPELY_NPM_START_SHORTCUT === '0')
|
|
101
|
+
return undefined;
|
|
102
|
+
if (Object.keys(env).some((k) => SCRIPT_CONFIG_ENV_RE.test(k)))
|
|
103
|
+
return undefined;
|
|
104
|
+
// `start [flags] [-- extra]` or `run|run-script <name> [flags] [-- extra]`
|
|
105
|
+
const dashDash = npmArgs.indexOf('--');
|
|
106
|
+
const head = dashDash === -1 ? npmArgs : npmArgs.slice(0, dashDash);
|
|
107
|
+
const extra = dashDash === -1 ? [] : npmArgs.slice(dashDash + 1);
|
|
108
|
+
const positional = head.filter((a) => !a.startsWith('-'));
|
|
109
|
+
if (head.some((a) => a.startsWith('-') && !QUIET_FLAGS.has(a)))
|
|
110
|
+
return undefined;
|
|
111
|
+
let scriptName;
|
|
112
|
+
if (positional.length === 1 && positional[0] === 'start') {
|
|
113
|
+
scriptName = 'start';
|
|
114
|
+
}
|
|
115
|
+
else if (positional.length === 2 && (positional[0] === 'run' || positional[0] === 'run-script')) {
|
|
116
|
+
scriptName = positional[1];
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
return undefined;
|
|
120
|
+
}
|
|
121
|
+
if (!extra.every((a) => PLAIN_TOKEN_RE.test(a)))
|
|
122
|
+
return undefined;
|
|
123
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
124
|
+
const fs = require('fs');
|
|
125
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
126
|
+
const path = require('path');
|
|
127
|
+
const cwd = options.cwd ?? process.cwd();
|
|
128
|
+
const packageJsonPath = path.join(cwd, 'package.json');
|
|
129
|
+
const execPath = options.execPath ?? process.execPath;
|
|
130
|
+
const npmCliPath = options.npmCliPath ?? process.argv[1];
|
|
131
|
+
// The npmrc files npm reads: builtin (npm's install dir), global
|
|
132
|
+
// ($prefix/etc/npmrc; prefix defaults to node's install dir unless an
|
|
133
|
+
// npmrc moves it), user, project.
|
|
134
|
+
const home = env.HOME || require('os').homedir();
|
|
135
|
+
const builtin = npmCliPath ? readNpmrc(path.join(path.dirname(path.dirname(npmCliPath)), 'npmrc')) : { affectsScripts: false };
|
|
136
|
+
const user = home ? readNpmrc(path.join(home, '.npmrc')) : { affectsScripts: false };
|
|
137
|
+
const project = readNpmrc(path.join(cwd, '.npmrc'));
|
|
138
|
+
const prefix = project.prefix ?? user.prefix ?? builtin.prefix ?? path.dirname(path.dirname(execPath));
|
|
139
|
+
const global = readNpmrc(path.join(prefix, 'etc', 'npmrc'));
|
|
140
|
+
if ([builtin, global, user, project].some((rc) => rc.affectsScripts))
|
|
141
|
+
return undefined;
|
|
142
|
+
let pkg;
|
|
143
|
+
try {
|
|
144
|
+
pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
if (!pkg || typeof pkg !== 'object' || pkg.workspaces)
|
|
150
|
+
return undefined;
|
|
151
|
+
const scripts = pkg.scripts ?? {};
|
|
152
|
+
const script = scripts[scriptName];
|
|
153
|
+
if (typeof script !== 'string')
|
|
154
|
+
return undefined;
|
|
155
|
+
if (scripts[`pre${scriptName}`] !== undefined || scripts[`post${scriptName}`] !== undefined)
|
|
156
|
+
return undefined;
|
|
157
|
+
const tokens = script.trim().split(/[ \t]+/);
|
|
158
|
+
if (tokens[0] !== 'node' || tokens.length < 2)
|
|
159
|
+
return undefined;
|
|
160
|
+
if (!tokens.every((t) => PLAIN_TOKEN_RE.test(t)))
|
|
161
|
+
return undefined;
|
|
162
|
+
const silent = head.some((a) => QUIET_FLAGS.has(a));
|
|
163
|
+
// The script environment npm provides (@npmcli/run-script): local bins on
|
|
164
|
+
// PATH, package metadata, lifecycle names.
|
|
165
|
+
const binDirs = [];
|
|
166
|
+
for (let dir = cwd;;) {
|
|
167
|
+
binDirs.push(path.join(dir, 'node_modules', '.bin'));
|
|
168
|
+
const parent = path.dirname(dir);
|
|
169
|
+
if (parent === dir)
|
|
170
|
+
break;
|
|
171
|
+
dir = parent;
|
|
172
|
+
}
|
|
173
|
+
// A local `node` binary would win on npm's PATH.
|
|
174
|
+
if (binDirs.some((dir) => fs.existsSync(path.join(dir, 'node'))))
|
|
175
|
+
return undefined;
|
|
176
|
+
const childEnv = {
|
|
177
|
+
...env,
|
|
178
|
+
PATH: [...binDirs, env.PATH].filter(Boolean).join(path.delimiter),
|
|
179
|
+
INIT_CWD: env.INIT_CWD ?? cwd,
|
|
180
|
+
NODE: execPath,
|
|
181
|
+
npm_node_execpath: execPath,
|
|
182
|
+
npm_lifecycle_event: scriptName,
|
|
183
|
+
npm_lifecycle_script: script,
|
|
184
|
+
npm_package_json: packageJsonPath,
|
|
185
|
+
npm_command: scriptName === 'start' ? 'start' : 'run-script',
|
|
186
|
+
};
|
|
187
|
+
if (npmCliPath)
|
|
188
|
+
childEnv.npm_execpath = npmCliPath;
|
|
189
|
+
if (typeof pkg.name === 'string')
|
|
190
|
+
childEnv.npm_package_name = pkg.name;
|
|
191
|
+
if (typeof pkg.version === 'string')
|
|
192
|
+
childEnv.npm_package_version = pkg.version;
|
|
193
|
+
if (silent)
|
|
194
|
+
childEnv.npm_config_loglevel = 'silent';
|
|
195
|
+
return { command: execPath, args: [...tokens.slice(1), ...extra], env: childEnv, cwd, scriptName };
|
|
196
|
+
}
|
|
197
|
+
const FORWARDED_SIGNALS = ['SIGTERM', 'SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR2'];
|
|
198
|
+
/**
|
|
199
|
+
* Run the planned node process in place of npm: inherit stdio, forward
|
|
200
|
+
* signals, exit with its exit code (128 + signal number when it was killed).
|
|
201
|
+
*
|
|
202
|
+
* @internal Exported for tests.
|
|
203
|
+
*/
|
|
204
|
+
function runNpmStartPlan(plan) {
|
|
205
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
206
|
+
const { spawn } = require('child_process');
|
|
207
|
+
const child = spawn(plan.command, plan.args, { cwd: plan.cwd, env: plan.env, stdio: 'inherit' });
|
|
208
|
+
const forward = (signal) => {
|
|
209
|
+
try {
|
|
210
|
+
child.kill(signal);
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
// already exited
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
for (const signal of FORWARDED_SIGNALS)
|
|
217
|
+
process.on(signal, forward);
|
|
218
|
+
child.on('error', (err) => {
|
|
219
|
+
console.error(`[scrapely] could not start '${plan.command} ${plan.args.join(' ')}': ${err.message}`);
|
|
220
|
+
process.exit(1);
|
|
221
|
+
});
|
|
222
|
+
child.on('exit', (code, signal) => {
|
|
223
|
+
for (const s of FORWARDED_SIGNALS)
|
|
224
|
+
process.removeListener(s, forward);
|
|
225
|
+
if (code !== null)
|
|
226
|
+
process.exit(code);
|
|
227
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
228
|
+
const signals = require('os').constants.signals;
|
|
229
|
+
process.exit(128 + (signal ? signals[signal] ?? 1 : 1));
|
|
230
|
+
});
|
|
231
|
+
return child;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Called by the preload in every Node process. Returns true for
|
|
235
|
+
* package-manager processes (no loader hooks needed). For a shortcut-able
|
|
236
|
+
* `npm start`, the main module load (npm-cli.js) is intercepted: the start
|
|
237
|
+
* script's node process runs instead and npm itself is never loaded. If Node
|
|
238
|
+
* ever loaded the main module another way, npm simply runs as before.
|
|
239
|
+
*/
|
|
240
|
+
function handlePackageManagerProcess() {
|
|
241
|
+
const manager = detectPackageManager();
|
|
242
|
+
if (!manager)
|
|
243
|
+
return false;
|
|
244
|
+
if (manager !== 'npm')
|
|
245
|
+
return true;
|
|
246
|
+
let plan;
|
|
247
|
+
try {
|
|
248
|
+
plan = planNpmStart();
|
|
249
|
+
}
|
|
250
|
+
catch {
|
|
251
|
+
plan = undefined;
|
|
252
|
+
}
|
|
253
|
+
if (!plan)
|
|
254
|
+
return true;
|
|
255
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
256
|
+
const Module = require('module');
|
|
257
|
+
const origLoad = Module._load;
|
|
258
|
+
if (typeof origLoad !== 'function')
|
|
259
|
+
return true;
|
|
260
|
+
const planned = plan;
|
|
261
|
+
Module._load = function (request, parent, isMain) {
|
|
262
|
+
if (isMain) {
|
|
263
|
+
Module._load = origLoad;
|
|
264
|
+
runNpmStartPlan(planned);
|
|
265
|
+
return {};
|
|
266
|
+
}
|
|
267
|
+
return origLoad.apply(this, [request, parent, isMain]);
|
|
268
|
+
};
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
@@ -59,8 +59,48 @@ export declare function patchCrawleeCrawlerModule(mod: any): void;
|
|
|
59
59
|
* @internal Exported for tests.
|
|
60
60
|
*/
|
|
61
61
|
export declare function patchLoadedCrawleeCopies(): void;
|
|
62
|
+
/**
|
|
63
|
+
* Patch the launcher classes exported by one loaded Crawlee launcher file (or
|
|
64
|
+
* package index). Idempotent.
|
|
65
|
+
*
|
|
66
|
+
* - `cdpUrl` becomes a valid launcher option (PlaywrightLauncher and
|
|
67
|
+
* PuppeteerLauncher copy BrowserLauncher's shape when their file loads, so
|
|
68
|
+
* each class gets it; browser-launcher.js loads first, so the copies inherit it).
|
|
69
|
+
* - Local launches get gVisor-safe Chrome args under runsc. Playwright/Puppeteer
|
|
70
|
+
* pool launches use CDP and skip this.
|
|
71
|
+
*
|
|
72
|
+
* @internal Exported for tests.
|
|
73
|
+
*/
|
|
74
|
+
export declare function patchBrowserLauncherModule(mod: any, parentFilename?: string): void;
|
|
62
75
|
/** @internal Exported for tests. */
|
|
63
76
|
export declare function patchRawBrowserType(chromium: {
|
|
64
77
|
connectOverCDP: (url: string, opts?: unknown) => Promise<any>;
|
|
65
78
|
}, browserType: any, name: string): void;
|
|
79
|
+
/**
|
|
80
|
+
* Patch puppeteer's PuppeteerNode class (puppeteer-core `node/PuppeteerNode.js`,
|
|
81
|
+
* shared by puppeteer and puppeteer-core, CJS and ESM builds). The package
|
|
82
|
+
* entry creates its default instance right after this class loads, and the
|
|
83
|
+
* constructor binds `launch`/`connect` from the prototype, so patching the
|
|
84
|
+
* prototype first also covers the named exports (`import { launch } from
|
|
85
|
+
* 'puppeteer'`), which are those bound methods. Idempotent per class.
|
|
86
|
+
*
|
|
87
|
+
* @internal Exported for tests.
|
|
88
|
+
*/
|
|
89
|
+
export declare function patchPuppeteerNodeClass(PuppeteerNode: any): void;
|
|
90
|
+
/**
|
|
91
|
+
* Patch copies that loaded before this module (e.g. a local run that imports
|
|
92
|
+
* crawlee or playwright before scrapely). Copies loaded later are patched by
|
|
93
|
+
* the loader hooks.
|
|
94
|
+
*
|
|
95
|
+
* @internal Exported for tests.
|
|
96
|
+
*/
|
|
97
|
+
export declare function patchLoadedModules(): void;
|
|
98
|
+
/** @internal Global the ESM load hook's appended code calls. */
|
|
99
|
+
export declare const PUPPETEER_NODE_PATCH_GLOBAL = "scrapely.patchPuppeteerNodeClass";
|
|
100
|
+
/** puppeteer-core's ESM PuppeteerNode.js (URL): lib/esm/puppeteer/... up to v24, lib/puppeteer/... (ESM only) from v25. */
|
|
101
|
+
export declare const PUPPETEER_NODE_ESM_URL_RE: RegExp;
|
|
102
|
+
/** Appended to the ESM PuppeteerNode.js source; runs right after the class is defined. */
|
|
103
|
+
export declare const PUPPETEER_NODE_ESM_SUFFIX: string;
|
|
104
|
+
/** @internal Whether an ESM import of puppeteer / puppeteer-core could succeed in this process. */
|
|
105
|
+
export declare function esmPuppeteerInstalled(): boolean;
|
|
66
106
|
export {};
|