scrapely 1.2.11 → 1.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +5 -0
- package/dist/actor.js +35 -5
- package/dist/cdp-retry.d.ts +43 -0
- package/dist/cdp-retry.js +171 -0
- package/dist/patch-browser-pool.js +8 -0
- package/dist/proxy-input-aliases.d.ts +15 -0
- package/dist/proxy-input-aliases.js +66 -0
- package/dist/storage/api-storage-client.js +19 -11
- package/package.json +6 -2
package/dist/actor.d.ts
CHANGED
|
@@ -517,6 +517,11 @@ export declare class Actor {
|
|
|
517
517
|
/** One-shot bounded reconciliation on fatal process errors. */
|
|
518
518
|
private static installFatalChargeHandlers;
|
|
519
519
|
private static _shutdownActor;
|
|
520
|
+
/**
|
|
521
|
+
* Store the exit/fail status message on the run as terminal. Best effort and
|
|
522
|
+
* bounded: a failing or slow API is logged and never blocks the exit.
|
|
523
|
+
*/
|
|
524
|
+
private static _persistTerminalStatusMessage;
|
|
520
525
|
/**
|
|
521
526
|
* Sets the status message for the current Actor run.
|
|
522
527
|
*
|
package/dist/actor.js
CHANGED
|
@@ -18,6 +18,7 @@ var ACTOR_PERMISSION_LEVEL;
|
|
|
18
18
|
})(ACTOR_PERMISSION_LEVEL || (exports.ACTOR_PERMISSION_LEVEL = ACTOR_PERMISSION_LEVEL = {}));
|
|
19
19
|
const core_1 = require("@crawlee/core");
|
|
20
20
|
const configuration_js_1 = require("./configuration.js");
|
|
21
|
+
const proxy_input_aliases_js_1 = require("./proxy-input-aliases.js");
|
|
21
22
|
const patch_storage_js_1 = require("./storage/patch-storage.js");
|
|
22
23
|
const api_storage_client_js_1 = require("./storage/api-storage-client.js");
|
|
23
24
|
const events_client_js_1 = require("./events/events-client.js");
|
|
@@ -382,9 +383,11 @@ class Actor {
|
|
|
382
383
|
const code = opts.exitCode ?? 0;
|
|
383
384
|
const timeoutSecs = opts.timeoutSecs ?? 30;
|
|
384
385
|
const shouldExit = opts.exit ?? true;
|
|
385
|
-
if (opts.statusMessage)
|
|
386
|
-
console.log(`[Scrapely] ${opts.statusMessage}`);
|
|
387
386
|
Actor._ensureActorInit('exit');
|
|
387
|
+
// Apify parity: the exit/fail message becomes the run's terminal status
|
|
388
|
+
// message. Stored before teardown, while the run and process are alive.
|
|
389
|
+
if (opts.statusMessage)
|
|
390
|
+
await Actor._persistTerminalStatusMessage(opts.statusMessage, code);
|
|
388
391
|
await Actor._shutdownActor(timeoutSecs);
|
|
389
392
|
if (shouldExit) {
|
|
390
393
|
await new Promise((resolve) => process.stdout.write('', () => resolve()));
|
|
@@ -415,7 +418,7 @@ class Actor {
|
|
|
415
418
|
// Same treatment as the regular input: decrypt secret fields
|
|
416
419
|
// and let the synchronous Actor.input getter return it.
|
|
417
420
|
if (typeof metamorphInput === 'object' && !Array.isArray(metamorphInput)) {
|
|
418
|
-
const decrypted = (0, secrets_js_1.decryptSecretInputs)(metamorphInput);
|
|
421
|
+
const decrypted = (0, proxy_input_aliases_js_1.withProxyKeyAliases)((0, secrets_js_1.decryptSecretInputs)(metamorphInput));
|
|
419
422
|
cachedInput = decrypted;
|
|
420
423
|
return decrypted;
|
|
421
424
|
}
|
|
@@ -446,7 +449,8 @@ class Actor {
|
|
|
446
449
|
// Decrypt secret inputs after loading (must happen here, not in init(),
|
|
447
450
|
// because the input needs to be available first)
|
|
448
451
|
if (input && typeof input === 'object' && !Array.isArray(input)) {
|
|
449
|
-
|
|
452
|
+
// Proxy objects carry both namings (scrapelyProxy* and apifyProxy*).
|
|
453
|
+
const decrypted = (0, proxy_input_aliases_js_1.withProxyKeyAliases)((0, secrets_js_1.decryptSecretInputs)(input));
|
|
450
454
|
// Large inputs are only in the INPUT record (no ACTOR_INPUT env):
|
|
451
455
|
// let the synchronous Actor.input getter return what was loaded here.
|
|
452
456
|
cachedInput = decrypted;
|
|
@@ -492,7 +496,7 @@ class Actor {
|
|
|
492
496
|
}
|
|
493
497
|
// Decrypt secret inputs after loading
|
|
494
498
|
if (cachedInput && typeof cachedInput === 'object' && !Array.isArray(cachedInput)) {
|
|
495
|
-
cachedInput = (0, secrets_js_1.decryptSecretInputs)(cachedInput);
|
|
499
|
+
cachedInput = (0, proxy_input_aliases_js_1.withProxyKeyAliases)((0, secrets_js_1.decryptSecretInputs)(cachedInput));
|
|
496
500
|
}
|
|
497
501
|
return cachedInput ?? undefined;
|
|
498
502
|
}
|
|
@@ -949,6 +953,31 @@ class Actor {
|
|
|
949
953
|
// drain, usually within milliseconds, and is still bounded.
|
|
950
954
|
await waitForStdioDrain(Math.min(1000, flushDeadlineMs));
|
|
951
955
|
}
|
|
956
|
+
/**
|
|
957
|
+
* Store the exit/fail status message on the run as terminal. Best effort and
|
|
958
|
+
* bounded: a failing or slow API is logged and never blocks the exit.
|
|
959
|
+
*/
|
|
960
|
+
static async _persistTerminalStatusMessage(message, exitCode, timeoutMs = 10_000) {
|
|
961
|
+
let timer;
|
|
962
|
+
try {
|
|
963
|
+
await Promise.race([
|
|
964
|
+
Actor.setStatusMessage(message, {
|
|
965
|
+
isStatusMessageTerminal: true,
|
|
966
|
+
level: exitCode > 0 ? 'ERROR' : 'INFO',
|
|
967
|
+
}),
|
|
968
|
+
new Promise((_, reject) => {
|
|
969
|
+
timer = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
970
|
+
}),
|
|
971
|
+
]);
|
|
972
|
+
}
|
|
973
|
+
catch (err) {
|
|
974
|
+
console.warn(`[Scrapely] Failed to store the final status message on the run: ${err.message}`);
|
|
975
|
+
}
|
|
976
|
+
finally {
|
|
977
|
+
if (timer)
|
|
978
|
+
clearTimeout(timer);
|
|
979
|
+
}
|
|
980
|
+
}
|
|
952
981
|
/**
|
|
953
982
|
* Sets the status message for the current Actor run.
|
|
954
983
|
*
|
|
@@ -1000,6 +1029,7 @@ class Actor {
|
|
|
1000
1029
|
statusMessage: message,
|
|
1001
1030
|
isStatusMessageTerminal,
|
|
1002
1031
|
}),
|
|
1032
|
+
signal: AbortSignal.timeout(10_000),
|
|
1003
1033
|
});
|
|
1004
1034
|
if (!response.ok) {
|
|
1005
1035
|
const error = await response.text();
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retry remote-browser (CDP) connects that fail for transient reasons.
|
|
3
|
+
*
|
|
4
|
+
* The remote browser answers 429 when it is at capacity and 503 when it cannot
|
|
5
|
+
* start a browser or validate the session right now, both with a Retry-After
|
|
6
|
+
* delay (repeated in the response body, because Playwright only puts the
|
|
7
|
+
* status line and body into its error message). Connection errors and
|
|
8
|
+
* timeouts are transient too. Those are retried with exponential backoff,
|
|
9
|
+
* waiting at least the Retry-After delay when the error carries one. Any other
|
|
10
|
+
* status (400/401/402/403/404) fails at once.
|
|
11
|
+
*
|
|
12
|
+
* Tuning (environment):
|
|
13
|
+
* - SCRAPELY_CDP_CONNECT_ATTEMPTS: total attempts, default 5 (1 disables retries).
|
|
14
|
+
* - SCRAPELY_CDP_RETRY_BASE_SECS: first backoff delay, default 1; doubles per attempt.
|
|
15
|
+
* - SCRAPELY_CDP_RETRY_MAX_SECS: cap for one backoff delay, default 30.
|
|
16
|
+
* - SCRAPELY_CDP_RETRY_AFTER_MAX_SECS: longest Retry-After the SDK waits out,
|
|
17
|
+
* default 120. A longer Retry-After ends the retries (the SDK never retries
|
|
18
|
+
* earlier than the server asked).
|
|
19
|
+
*/
|
|
20
|
+
/** @internal Exported for tests. */
|
|
21
|
+
export declare function classifyConnectError(err: unknown): {
|
|
22
|
+
retryable: boolean;
|
|
23
|
+
status?: number;
|
|
24
|
+
retryAfterSecs?: number;
|
|
25
|
+
};
|
|
26
|
+
/** @internal Delay in ms before retry number `attempt` (1-based), honoring Retry-After. */
|
|
27
|
+
export declare function backoffDelayMs(attempt: number, retryAfterSecs?: number): number;
|
|
28
|
+
/** @internal Overridable for tests. */
|
|
29
|
+
export declare const retryTiming: {
|
|
30
|
+
sleep: (ms: number) => Promise<void>;
|
|
31
|
+
};
|
|
32
|
+
/** Run a CDP connect, retrying transient failures with backoff. */
|
|
33
|
+
export declare function connectWithRetry<T>(connect: () => Promise<T>): Promise<T>;
|
|
34
|
+
/** True when `url` points at the same host as the platform's CDP_URL. */
|
|
35
|
+
export declare function isPlatformCdpUrl(url: unknown, platformCdpUrl: string | undefined): boolean;
|
|
36
|
+
export declare function isRetryWrapped(fn: unknown): boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Make `target[method]` (Playwright `connectOverCDP` or Puppeteer `connect`)
|
|
39
|
+
* retry transient failures when it connects to the platform CDP host. Covers
|
|
40
|
+
* actor code that connects to CDP_URL directly as well as the SDK's own
|
|
41
|
+
* redirected launches. Other hosts are left alone. Idempotent.
|
|
42
|
+
*/
|
|
43
|
+
export declare function wrapConnectWithRetry(target: any, method: 'connectOverCDP' | 'connect', getPlatformCdpUrl: () => string | undefined): void;
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Retry remote-browser (CDP) connects that fail for transient reasons.
|
|
4
|
+
*
|
|
5
|
+
* The remote browser answers 429 when it is at capacity and 503 when it cannot
|
|
6
|
+
* start a browser or validate the session right now, both with a Retry-After
|
|
7
|
+
* delay (repeated in the response body, because Playwright only puts the
|
|
8
|
+
* status line and body into its error message). Connection errors and
|
|
9
|
+
* timeouts are transient too. Those are retried with exponential backoff,
|
|
10
|
+
* waiting at least the Retry-After delay when the error carries one. Any other
|
|
11
|
+
* status (400/401/402/403/404) fails at once.
|
|
12
|
+
*
|
|
13
|
+
* Tuning (environment):
|
|
14
|
+
* - SCRAPELY_CDP_CONNECT_ATTEMPTS: total attempts, default 5 (1 disables retries).
|
|
15
|
+
* - SCRAPELY_CDP_RETRY_BASE_SECS: first backoff delay, default 1; doubles per attempt.
|
|
16
|
+
* - SCRAPELY_CDP_RETRY_MAX_SECS: cap for one backoff delay, default 30.
|
|
17
|
+
* - SCRAPELY_CDP_RETRY_AFTER_MAX_SECS: longest Retry-After the SDK waits out,
|
|
18
|
+
* default 120. A longer Retry-After ends the retries (the SDK never retries
|
|
19
|
+
* earlier than the server asked).
|
|
20
|
+
*/
|
|
21
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.retryTiming = void 0;
|
|
23
|
+
exports.classifyConnectError = classifyConnectError;
|
|
24
|
+
exports.backoffDelayMs = backoffDelayMs;
|
|
25
|
+
exports.connectWithRetry = connectWithRetry;
|
|
26
|
+
exports.isPlatformCdpUrl = isPlatformCdpUrl;
|
|
27
|
+
exports.isRetryWrapped = isRetryWrapped;
|
|
28
|
+
exports.wrapConnectWithRetry = wrapConnectWithRetry;
|
|
29
|
+
const RETRY_WRAPPED = Symbol.for('scrapely.cdpRetryWrapped');
|
|
30
|
+
const REASONS = 'Too Many Requests|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout|' +
|
|
31
|
+
'Request Timeout|Bad Request|Unauthorized|Payment Required|Forbidden|Not Found';
|
|
32
|
+
const STATUS_RE = new RegExp(`\\b([1-5]\\d\\d) (?:${REASONS})\\b`);
|
|
33
|
+
const WS_STATUS_RE = /Unexpected server response:?\s*(\d{3})/i;
|
|
34
|
+
const RETRY_AFTER_RE = /Retry-After:?\s*(\d+(?:\.\d+)?)/i;
|
|
35
|
+
const TRANSIENT_MARKERS = [
|
|
36
|
+
'econnrefused',
|
|
37
|
+
'econnreset',
|
|
38
|
+
'etimedout',
|
|
39
|
+
'epipe',
|
|
40
|
+
'eai_again',
|
|
41
|
+
'ehostunreach',
|
|
42
|
+
'enetunreach',
|
|
43
|
+
'socket hang up',
|
|
44
|
+
'timeout',
|
|
45
|
+
'timed out',
|
|
46
|
+
'connection closed',
|
|
47
|
+
'connection refused',
|
|
48
|
+
'connection reset',
|
|
49
|
+
'websocket error',
|
|
50
|
+
'net::err_',
|
|
51
|
+
'target closed',
|
|
52
|
+
];
|
|
53
|
+
function envNumber(name, fallback) {
|
|
54
|
+
const value = Number(process.env[name]);
|
|
55
|
+
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
56
|
+
}
|
|
57
|
+
function maxAttempts() {
|
|
58
|
+
return Math.max(1, Math.floor(envNumber('SCRAPELY_CDP_CONNECT_ATTEMPTS', 5)));
|
|
59
|
+
}
|
|
60
|
+
function errorText(err) {
|
|
61
|
+
if (err instanceof Error) {
|
|
62
|
+
const code = err.code;
|
|
63
|
+
return `${err.name}: ${err.message}${code ? ` ${code}` : ''}`;
|
|
64
|
+
}
|
|
65
|
+
return String(err);
|
|
66
|
+
}
|
|
67
|
+
/** @internal Exported for tests. */
|
|
68
|
+
function classifyConnectError(err) {
|
|
69
|
+
const text = errorText(err);
|
|
70
|
+
const statusMatch = WS_STATUS_RE.exec(text) ?? STATUS_RE.exec(text);
|
|
71
|
+
const retryAfterMatch = RETRY_AFTER_RE.exec(text);
|
|
72
|
+
const retryAfterSecs = retryAfterMatch ? Number(retryAfterMatch[1]) : undefined;
|
|
73
|
+
if (statusMatch) {
|
|
74
|
+
const status = Number(statusMatch[1]);
|
|
75
|
+
return { retryable: status === 408 || status === 429 || status >= 500, status, retryAfterSecs };
|
|
76
|
+
}
|
|
77
|
+
const lowered = text.toLowerCase();
|
|
78
|
+
return { retryable: TRANSIENT_MARKERS.some((m) => lowered.includes(m)), retryAfterSecs };
|
|
79
|
+
}
|
|
80
|
+
/** @internal Delay in ms before retry number `attempt` (1-based), honoring Retry-After. */
|
|
81
|
+
function backoffDelayMs(attempt, retryAfterSecs) {
|
|
82
|
+
const base = envNumber('SCRAPELY_CDP_RETRY_BASE_SECS', 1);
|
|
83
|
+
const cap = envNumber('SCRAPELY_CDP_RETRY_MAX_SECS', 30);
|
|
84
|
+
let delay = Math.min(cap, base * 2 ** (attempt - 1)) * (0.75 + Math.random() * 0.5);
|
|
85
|
+
if (retryAfterSecs !== undefined)
|
|
86
|
+
delay = Math.max(delay, retryAfterSecs);
|
|
87
|
+
return Math.round(delay * 1000);
|
|
88
|
+
}
|
|
89
|
+
function describe(err, status) {
|
|
90
|
+
if (status !== undefined)
|
|
91
|
+
return `HTTP ${status}`;
|
|
92
|
+
// First line only, URL masked: connect errors can carry the session URL with ids.
|
|
93
|
+
const first = (err instanceof Error ? err.message : String(err)).trim().split('\n')[0] || 'error';
|
|
94
|
+
return first.replace(/(wss?|https?):\/\/\S+/g, '<cdp-url>').slice(0, 200);
|
|
95
|
+
}
|
|
96
|
+
/** @internal Overridable for tests. */
|
|
97
|
+
exports.retryTiming = {
|
|
98
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
99
|
+
};
|
|
100
|
+
/** Run a CDP connect, retrying transient failures with backoff. */
|
|
101
|
+
async function connectWithRetry(connect) {
|
|
102
|
+
const attempts = maxAttempts();
|
|
103
|
+
for (let attempt = 1;; attempt++) {
|
|
104
|
+
try {
|
|
105
|
+
return await connect();
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
const { retryable, status, retryAfterSecs } = classifyConnectError(err);
|
|
109
|
+
if (!retryable || attempt >= attempts)
|
|
110
|
+
throw err;
|
|
111
|
+
if (retryAfterSecs !== undefined && retryAfterSecs > envNumber('SCRAPELY_CDP_RETRY_AFTER_MAX_SECS', 120)) {
|
|
112
|
+
// Never retry earlier than the server asked; give up instead of waiting that long.
|
|
113
|
+
throw err;
|
|
114
|
+
}
|
|
115
|
+
const delayMs = backoffDelayMs(attempt, retryAfterSecs);
|
|
116
|
+
console.warn(`[scrapely] Remote browser connect failed (${describe(err, status)}); ` +
|
|
117
|
+
`retrying in ${(delayMs / 1000).toFixed(1)}s (attempt ${attempt + 1}/${attempts})`);
|
|
118
|
+
await exports.retryTiming.sleep(delayMs);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/** True when `url` points at the same host as the platform's CDP_URL. */
|
|
123
|
+
function isPlatformCdpUrl(url, platformCdpUrl) {
|
|
124
|
+
if (typeof url !== 'string' || !platformCdpUrl)
|
|
125
|
+
return false;
|
|
126
|
+
try {
|
|
127
|
+
const target = new URL(url);
|
|
128
|
+
const platform = new URL(platformCdpUrl);
|
|
129
|
+
return !!target.host && target.host.toLowerCase() === platform.host.toLowerCase();
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
function connectTarget(arg) {
|
|
136
|
+
if (typeof arg === 'string')
|
|
137
|
+
return arg;
|
|
138
|
+
if (arg && typeof arg === 'object') {
|
|
139
|
+
const o = arg;
|
|
140
|
+
return o.endpointURL ?? o.wsEndpoint ?? o.browserWSEndpoint ?? o.browserURL;
|
|
141
|
+
}
|
|
142
|
+
return undefined;
|
|
143
|
+
}
|
|
144
|
+
function isRetryWrapped(fn) {
|
|
145
|
+
return typeof fn === 'function' && fn[RETRY_WRAPPED] === true;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Make `target[method]` (Playwright `connectOverCDP` or Puppeteer `connect`)
|
|
149
|
+
* retry transient failures when it connects to the platform CDP host. Covers
|
|
150
|
+
* actor code that connects to CDP_URL directly as well as the SDK's own
|
|
151
|
+
* redirected launches. Other hosts are left alone. Idempotent.
|
|
152
|
+
*/
|
|
153
|
+
function wrapConnectWithRetry(target, method, getPlatformCdpUrl) {
|
|
154
|
+
const original = target?.[method];
|
|
155
|
+
if (typeof original !== 'function' || isRetryWrapped(original))
|
|
156
|
+
return;
|
|
157
|
+
const wrapped = function (...args) {
|
|
158
|
+
const self = this ?? target;
|
|
159
|
+
if (!isPlatformCdpUrl(connectTarget(args[0]), getPlatformCdpUrl())) {
|
|
160
|
+
return original.apply(self, args);
|
|
161
|
+
}
|
|
162
|
+
return connectWithRetry(() => original.apply(self, args));
|
|
163
|
+
};
|
|
164
|
+
wrapped[RETRY_WRAPPED] = true;
|
|
165
|
+
try {
|
|
166
|
+
target[method] = wrapped;
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
// frozen object: leave it unwrapped
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -18,6 +18,7 @@ exports.patchRawBrowserType = patchRawBrowserType;
|
|
|
18
18
|
*/
|
|
19
19
|
const browser_configuration_js_1 = require("./browser-configuration.js");
|
|
20
20
|
const browser_locations_js_1 = require("./browser-locations.js");
|
|
21
|
+
const cdp_retry_js_1 = require("./cdp-retry.js");
|
|
21
22
|
let _cdpUrl;
|
|
22
23
|
/**
|
|
23
24
|
* Lazily get CDP URL from environment or cached value.
|
|
@@ -321,6 +322,7 @@ function patchPlaywrightPluginClass(PlaywrightPlugin) {
|
|
|
321
322
|
}
|
|
322
323
|
const cdpUrl = applyProxyInfoToCdpUrl(requireCdpUrl(), proxyInfoFromLaunchContext(launchContext), launchContext?.browserConfiguration);
|
|
323
324
|
const chromium = getPlaywrightChromium(this.library);
|
|
325
|
+
(0, cdp_retry_js_1.wrapConnectWithRetry)(chromium, 'connectOverCDP', getCdpUrl);
|
|
324
326
|
return chromium.connectOverCDP(cdpUrl);
|
|
325
327
|
};
|
|
326
328
|
proto._scrapelyCdpPatched = true;
|
|
@@ -344,6 +346,7 @@ function patchPuppeteerPluginClass(PuppeteerPlugin) {
|
|
|
344
346
|
if (!launcher || typeof launcher.connect !== 'function') {
|
|
345
347
|
throw new Error('Puppeteer library is missing connect(). Install puppeteer or puppeteer-core.');
|
|
346
348
|
}
|
|
349
|
+
(0, cdp_retry_js_1.wrapConnectWithRetry)(launcher, 'connect', getCdpUrl);
|
|
347
350
|
return launcher.connect({ browserWSEndpoint: cdpUrl });
|
|
348
351
|
};
|
|
349
352
|
proto._scrapelyCdpPatched = true;
|
|
@@ -669,6 +672,10 @@ function patchRawBrowserType(chromium, browserType, name) {
|
|
|
669
672
|
function patchRawPlaywrightModule(pw) {
|
|
670
673
|
if (!pw?.chromium || typeof pw.chromium.connectOverCDP !== 'function')
|
|
671
674
|
return;
|
|
675
|
+
// Retry transient connect failures (429/5xx/network) for the platform CDP
|
|
676
|
+
// host: covers direct chromium.connectOverCDP(CDP_URL) calls in actor code
|
|
677
|
+
// and the redirected launches, which call this same property.
|
|
678
|
+
(0, cdp_retry_js_1.wrapConnectWithRetry)(pw.chromium, 'connectOverCDP', getCdpUrl);
|
|
672
679
|
for (const name of ['chromium', 'firefox', 'webkit']) {
|
|
673
680
|
patchRawBrowserType(pw.chromium, pw[name], name);
|
|
674
681
|
}
|
|
@@ -683,6 +690,7 @@ function patchRawPuppeteerModule(mod, moduleName) {
|
|
|
683
690
|
if (typeof launcher?.launch !== 'function' || launcher._scrapelyRawCdpPatched) {
|
|
684
691
|
return;
|
|
685
692
|
}
|
|
693
|
+
(0, cdp_retry_js_1.wrapConnectWithRetry)(launcher, 'connect', getCdpUrl);
|
|
686
694
|
const origLaunch = launcher.launch.bind(launcher);
|
|
687
695
|
launcher.launch = async function rawPuppeteerLaunchPatched(launchOptions = {}) {
|
|
688
696
|
const { options, browserConfiguration } = takeBrowserConfiguration(launchOptions);
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Proxy input objects with both namings. An input-schema proxy field may be
|
|
3
|
+
* written with the Scrapely names (`useScrapelyProxy`, `scrapelyProxyGroups`,
|
|
4
|
+
* `scrapelyProxyCountry`, `scrapelyProxySubdivision`) or the Apify names
|
|
5
|
+
* (`useApifyProxy`, `apifyProxy*`). Actor.getInput() and Actor.input return
|
|
6
|
+
* every set option under both names, so actor code that reads either naming
|
|
7
|
+
* sees the user's choice.
|
|
8
|
+
*
|
|
9
|
+
* Precedence matches createProxyConfiguration(): a boolean `useScrapelyProxy`
|
|
10
|
+
* wins over `useApifyProxy`, and a set `scrapelyProxy*` value wins over its
|
|
11
|
+
* `apifyProxy*` counterpart. Nothing is removed: keys with unset or
|
|
12
|
+
* non-boolean values stay as sent.
|
|
13
|
+
*/
|
|
14
|
+
/** `input` with every top-level proxy object under both namings; `input` itself when nothing changes. Never mutates. */
|
|
15
|
+
export declare function withProxyKeyAliases<T>(input: T): T;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Proxy input objects with both namings. An input-schema proxy field may be
|
|
4
|
+
* written with the Scrapely names (`useScrapelyProxy`, `scrapelyProxyGroups`,
|
|
5
|
+
* `scrapelyProxyCountry`, `scrapelyProxySubdivision`) or the Apify names
|
|
6
|
+
* (`useApifyProxy`, `apifyProxy*`). Actor.getInput() and Actor.input return
|
|
7
|
+
* every set option under both names, so actor code that reads either naming
|
|
8
|
+
* sees the user's choice.
|
|
9
|
+
*
|
|
10
|
+
* Precedence matches createProxyConfiguration(): a boolean `useScrapelyProxy`
|
|
11
|
+
* wins over `useApifyProxy`, and a set `scrapelyProxy*` value wins over its
|
|
12
|
+
* `apifyProxy*` counterpart. Nothing is removed: keys with unset or
|
|
13
|
+
* non-boolean values stay as sent.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.withProxyKeyAliases = withProxyKeyAliases;
|
|
17
|
+
const OPTION_NAMES = ['Groups', 'Country', 'Subdivision'];
|
|
18
|
+
const SWITCH_KEYS = ['useScrapelyProxy', 'useApifyProxy'];
|
|
19
|
+
function isPlainObject(value) {
|
|
20
|
+
return !!value && typeof value === 'object' && !Array.isArray(value);
|
|
21
|
+
}
|
|
22
|
+
function isSet(value) {
|
|
23
|
+
return value !== undefined && value !== null;
|
|
24
|
+
}
|
|
25
|
+
function isProxyObject(value) {
|
|
26
|
+
return SWITCH_KEYS.some((k) => Object.hasOwn(value, k))
|
|
27
|
+
|| OPTION_NAMES.some((n) => Object.hasOwn(value, `scrapelyProxy${n}`) || Object.hasOwn(value, `apifyProxy${n}`));
|
|
28
|
+
}
|
|
29
|
+
function aliasProxyObject(value) {
|
|
30
|
+
const result = { ...value };
|
|
31
|
+
const use = SWITCH_KEYS.map((k) => value[k]).find((v) => typeof v === 'boolean');
|
|
32
|
+
if (use !== undefined) {
|
|
33
|
+
for (const k of SWITCH_KEYS)
|
|
34
|
+
result[k] = use;
|
|
35
|
+
}
|
|
36
|
+
for (const name of OPTION_NAMES) {
|
|
37
|
+
const scrapely = `scrapelyProxy${name}`;
|
|
38
|
+
const apify = `apifyProxy${name}`;
|
|
39
|
+
const option = isSet(value[scrapely]) ? value[scrapely] : value[apify];
|
|
40
|
+
if (isSet(option)) {
|
|
41
|
+
result[scrapely] = option;
|
|
42
|
+
result[apify] = option;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return result;
|
|
46
|
+
}
|
|
47
|
+
function sameObject(a, b) {
|
|
48
|
+
const keys = Object.keys(a);
|
|
49
|
+
return keys.length === Object.keys(b).length && keys.every((k) => Object.hasOwn(b, k) && a[k] === b[k]);
|
|
50
|
+
}
|
|
51
|
+
/** `input` with every top-level proxy object under both namings; `input` itself when nothing changes. Never mutates. */
|
|
52
|
+
function withProxyKeyAliases(input) {
|
|
53
|
+
if (!isPlainObject(input))
|
|
54
|
+
return input;
|
|
55
|
+
let result;
|
|
56
|
+
for (const [key, value] of Object.entries(input)) {
|
|
57
|
+
if (!isPlainObject(value) || !isProxyObject(value))
|
|
58
|
+
continue;
|
|
59
|
+
const aliased = aliasProxyObject(value);
|
|
60
|
+
if (sameObject(aliased, value))
|
|
61
|
+
continue;
|
|
62
|
+
result ??= { ...input };
|
|
63
|
+
result[key] = aliased;
|
|
64
|
+
}
|
|
65
|
+
return (result ?? input);
|
|
66
|
+
}
|
|
@@ -313,7 +313,10 @@ class BaseApiClient {
|
|
|
313
313
|
const response = await fetch(url, {
|
|
314
314
|
method,
|
|
315
315
|
headers,
|
|
316
|
-
|
|
316
|
+
// Binary bodies (KV records) go out as the raw bytes, never JSON/base64.
|
|
317
|
+
body: body === undefined ? undefined
|
|
318
|
+
: typeof body === 'string' || body instanceof Uint8Array ? body
|
|
319
|
+
: JSON.stringify(body),
|
|
317
320
|
signal: controller.signal,
|
|
318
321
|
});
|
|
319
322
|
// Update rate limiter state from response headers
|
|
@@ -429,14 +432,15 @@ class BaseApiClient {
|
|
|
429
432
|
return { data: undefined, contentType: '' };
|
|
430
433
|
}
|
|
431
434
|
const responseContentType = response.headers.get('content-type') || 'application/json';
|
|
432
|
-
|
|
433
|
-
|
|
435
|
+
const mime = responseContentType.split(';')[0].trim().toLowerCase() || 'application/json';
|
|
436
|
+
if (mime === 'application/json' || mime.endsWith('+json')) {
|
|
437
|
+
return { data: await response.json(), contentType: mime };
|
|
434
438
|
}
|
|
435
|
-
if (
|
|
436
|
-
return { data:
|
|
439
|
+
if (mime.startsWith('text/')) {
|
|
440
|
+
return { data: await response.text(), contentType: mime };
|
|
437
441
|
}
|
|
438
|
-
|
|
439
|
-
return { data:
|
|
442
|
+
// Any other content type (video/mp4, image/png, application/zip, ...) is binary.
|
|
443
|
+
return { data: Buffer.from(await response.arrayBuffer()), contentType: mime };
|
|
440
444
|
});
|
|
441
445
|
}
|
|
442
446
|
}
|
|
@@ -632,12 +636,16 @@ class ApiKeyValueStoreClient extends BaseApiClient {
|
|
|
632
636
|
for await (const chunk of value) {
|
|
633
637
|
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
634
638
|
}
|
|
635
|
-
|
|
636
|
-
|
|
639
|
+
body = Buffer.concat(chunks);
|
|
640
|
+
ct = 'application/octet-stream';
|
|
641
|
+
}
|
|
642
|
+
else if (value instanceof Uint8Array) {
|
|
643
|
+
// Raw bytes, as Apify stores them; the API returns them unchanged on GET.
|
|
644
|
+
body = value;
|
|
637
645
|
ct = 'application/octet-stream';
|
|
638
646
|
}
|
|
639
|
-
else if (
|
|
640
|
-
body =
|
|
647
|
+
else if (value instanceof ArrayBuffer) {
|
|
648
|
+
body = Buffer.from(value);
|
|
641
649
|
ct = 'application/octet-stream';
|
|
642
650
|
}
|
|
643
651
|
else if (typeof value === 'string') {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scrapely",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.13",
|
|
4
4
|
"description": "Scrapely SDK for building JavaScript and TypeScript Actors",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -36,7 +36,11 @@
|
|
|
36
36
|
"test:init-rollback": "npx tsx --test tests/init-rollback.test.ts",
|
|
37
37
|
"test:apify-parity": "npx tsx --test tests/apify-parity.test.ts",
|
|
38
38
|
"test:run-control": "npx tsx --test tests/run-control.test.ts",
|
|
39
|
-
"test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts"
|
|
39
|
+
"test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts",
|
|
40
|
+
"test:terminal-status": "npx tsx --test tests/terminal-status-message.test.ts",
|
|
41
|
+
"test:cdp-retry": "npx tsx --test tests/cdp-retry.test.ts",
|
|
42
|
+
"test:proxy-input-aliases": "npx tsx --test tests/proxy-input-aliases.test.ts",
|
|
43
|
+
"test:charge-retries": "npx tsx --test tests/charge-retries.test.ts"
|
|
40
44
|
},
|
|
41
45
|
"author": "SandeepDev1",
|
|
42
46
|
"license": "ISC",
|