scrapely 1.2.7 → 1.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/actor.d.ts
CHANGED
|
@@ -546,8 +546,8 @@ export declare class Actor {
|
|
|
546
546
|
* });
|
|
547
547
|
* ```
|
|
548
548
|
*
|
|
549
|
-
*
|
|
550
|
-
* `undefined` when `{
|
|
549
|
+
* Pass the proxy input field (`editor: "proxy"`) as is: this function returns
|
|
550
|
+
* `undefined` when `{ useScrapelyProxy: false }` is passed without custom proxies.
|
|
551
551
|
*
|
|
552
552
|
* @param options - Proxy configuration options
|
|
553
553
|
* @returns Promise resolving to ProxyConfiguration instance, or undefined
|
package/dist/actor.js
CHANGED
|
@@ -1015,8 +1015,8 @@ class Actor {
|
|
|
1015
1015
|
* });
|
|
1016
1016
|
* ```
|
|
1017
1017
|
*
|
|
1018
|
-
*
|
|
1019
|
-
* `undefined` when `{
|
|
1018
|
+
* Pass the proxy input field (`editor: "proxy"`) as is: this function returns
|
|
1019
|
+
* `undefined` when `{ useScrapelyProxy: false }` is passed without custom proxies.
|
|
1020
1020
|
*
|
|
1021
1021
|
* @param options - Proxy configuration options
|
|
1022
1022
|
* @returns Promise resolving to ProxyConfiguration instance, or undefined
|
|
@@ -53,14 +53,17 @@ export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions
|
|
|
53
53
|
/**
|
|
54
54
|
* Input-schema alias for groups. Normalized to groups.
|
|
55
55
|
*/
|
|
56
|
-
|
|
56
|
+
scrapelyProxyGroups?: string[];
|
|
57
57
|
/**
|
|
58
58
|
* Input-schema alias for subdivisionCode. Normalized to subdivisionCode.
|
|
59
59
|
*/
|
|
60
|
-
|
|
60
|
+
scrapelyProxySubdivision?: string;
|
|
61
61
|
/**
|
|
62
62
|
* Input-schema alias for countryCode. Normalized to countryCode.
|
|
63
63
|
*/
|
|
64
|
+
scrapelyProxyCountry?: string;
|
|
65
|
+
apifyProxyGroups?: string[];
|
|
66
|
+
apifyProxySubdivision?: string;
|
|
64
67
|
apifyProxyCountry?: string;
|
|
65
68
|
/**
|
|
66
69
|
* Platform proxy password. Defaults to the password injected by the
|
|
@@ -73,9 +76,10 @@ export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions
|
|
|
73
76
|
*/
|
|
74
77
|
export interface CreateProxyConfigurationOptions extends ProxyConfigurationOptions {
|
|
75
78
|
/**
|
|
76
|
-
*
|
|
77
|
-
*
|
|
79
|
+
* The proxy input field's switch (input schema `editor: "proxy"`). When
|
|
80
|
+
* `false` and no custom proxies are given, this method returns `undefined`.
|
|
78
81
|
*/
|
|
82
|
+
useScrapelyProxy?: boolean;
|
|
79
83
|
useApifyProxy?: boolean;
|
|
80
84
|
/**
|
|
81
85
|
* Whether to verify the proxy configuration by testing a connection.
|
|
@@ -39,14 +39,17 @@ function hasCustomProxyOptions(options) {
|
|
|
39
39
|
}
|
|
40
40
|
function normalizeProxyInput(options) {
|
|
41
41
|
const normalized = { ...options };
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
const groups = normalized.scrapelyProxyGroups ?? normalized.apifyProxyGroups;
|
|
43
|
+
if (groups && !normalized.groups) {
|
|
44
|
+
normalized.groups = groups;
|
|
44
45
|
}
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
const countryCode = normalized.scrapelyProxyCountry ?? normalized.apifyProxyCountry;
|
|
47
|
+
if (countryCode && !normalized.countryCode) {
|
|
48
|
+
normalized.countryCode = countryCode;
|
|
47
49
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
+
const subdivisionCode = normalized.scrapelyProxySubdivision ?? normalized.apifyProxySubdivision;
|
|
51
|
+
if (subdivisionCode && !normalized.subdivisionCode) {
|
|
52
|
+
normalized.subdivisionCode = subdivisionCode;
|
|
50
53
|
}
|
|
51
54
|
return normalized;
|
|
52
55
|
}
|
|
@@ -60,17 +63,19 @@ function normalizeProxyInput(options) {
|
|
|
60
63
|
*/
|
|
61
64
|
async function createProxyConfiguration(options = {}) {
|
|
62
65
|
const normalized = normalizeProxyInput(options);
|
|
63
|
-
const { useApifyProxy, checkAccess = true, ...proxyOptions } = normalized;
|
|
64
|
-
//
|
|
66
|
+
const { useScrapelyProxy, useApifyProxy, checkAccess = true, ...proxyOptions } = normalized;
|
|
67
|
+
// `useScrapelyProxy: false` (or the legacy `useApifyProxy: false`; a boolean
|
|
68
|
+
// useScrapelyProxy wins) explicitly disables the PLATFORM proxy
|
|
65
69
|
// only — custom proxies from the same input must still be used. The standard
|
|
66
|
-
// "Custom proxy URLs" input shape is `{
|
|
70
|
+
// "Custom proxy URLs" input shape is `{ useScrapelyProxy: false, proxyUrls: [...] }`;
|
|
67
71
|
// returning undefined for it would silently drop the user's own proxies and
|
|
68
72
|
// crawl on their real IPs. We only bail when NO custom proxy mode is given —
|
|
69
73
|
// proxyUrls, tieredProxyUrls, tieredProxyConfig and newUrlFunction all count
|
|
70
74
|
// (Python SDK Actor.create_proxy_configuration parity).
|
|
71
|
-
const
|
|
75
|
+
const usePlatformProxy = typeof useScrapelyProxy === 'boolean' ? useScrapelyProxy : useApifyProxy;
|
|
76
|
+
const dontUsePlatformProxy = usePlatformProxy === false;
|
|
72
77
|
const dontUseCustomProxies = !hasCustomProxyOptions(proxyOptions);
|
|
73
|
-
if (
|
|
78
|
+
if (dontUsePlatformProxy && dontUseCustomProxies) {
|
|
74
79
|
return undefined;
|
|
75
80
|
}
|
|
76
81
|
if (!hasCustomProxyOptions(proxyOptions)) {
|
|
@@ -89,7 +89,10 @@ export declare class ProxyConfiguration extends CrawleeProxyConfiguration {
|
|
|
89
89
|
/**
|
|
90
90
|
* Verifies the proxy configuration. For platform proxy mode with an
|
|
91
91
|
* injected status URL, performs the Apify-format access check through the
|
|
92
|
-
* gateway (`GET {statusUrl}/?format=json
|
|
92
|
+
* gateway (`GET {statusUrl}/?format=json`). When the gateway answers
|
|
93
|
+
* `connected: false` (e.g. an unknown proxy group), it throws with the
|
|
94
|
+
* gateway's `connectionError` on the platform and warns locally, like
|
|
95
|
+
* Apify's `_checkAccess`; timeouts and HTTP errors only warn.
|
|
93
96
|
* BYOP mode only verifies that a test URL can be generated.
|
|
94
97
|
*/
|
|
95
98
|
initialize(options?: {
|
|
@@ -73,6 +73,12 @@ function validateProxySessionId(sessionId) {
|
|
|
73
73
|
}
|
|
74
74
|
return sessionId;
|
|
75
75
|
}
|
|
76
|
+
/** `groups`, else the input-schema alias `scrapelyProxyGroups`, else the legacy `apifyProxyGroups`. */
|
|
77
|
+
function platformGroups(options) {
|
|
78
|
+
if (options.groups?.length)
|
|
79
|
+
return options.groups;
|
|
80
|
+
return options.scrapelyProxyGroups ?? options.apifyProxyGroups;
|
|
81
|
+
}
|
|
76
82
|
/**
|
|
77
83
|
* Resolves and validates the country/subdivision options (Apify parity).
|
|
78
84
|
* Country must be a two-letter ISO code; subdivision an ISO 3166-2 code
|
|
@@ -80,8 +86,8 @@ function validateProxySessionId(sessionId) {
|
|
|
80
86
|
* the normalized values used by the username grammar.
|
|
81
87
|
*/
|
|
82
88
|
function resolveCountryOptions(options) {
|
|
83
|
-
const countryCode = options.countryCode ?? options.apifyProxyCountry;
|
|
84
|
-
const subdivisionCode = options.subdivisionCode ?? options.apifyProxySubdivision;
|
|
89
|
+
const countryCode = options.countryCode ?? options.scrapelyProxyCountry ?? options.apifyProxyCountry;
|
|
90
|
+
const subdivisionCode = options.subdivisionCode ?? options.scrapelyProxySubdivision ?? options.apifyProxySubdivision;
|
|
85
91
|
if (countryCode != null && !COUNTRY_CODE_REGEX.test(countryCode)) {
|
|
86
92
|
throw new Error(`ProxyConfiguration: "countryCode" must be a two-letter ISO country code (got "${countryCode}").`);
|
|
87
93
|
}
|
|
@@ -225,7 +231,7 @@ async function requestThroughProxy(targetUrl, proxyUrl, timeoutMs) {
|
|
|
225
231
|
* server.
|
|
226
232
|
*/
|
|
227
233
|
function buildPlatformProxyUrl(options, env, sessionId) {
|
|
228
|
-
const groups = options
|
|
234
|
+
const groups = platformGroups(options);
|
|
229
235
|
const { countryCode, subdivisionCode } = resolveCountryOptions(options);
|
|
230
236
|
const password = options.password ?? env.password;
|
|
231
237
|
// Apify username grammar: groups, then session, then country — with the
|
|
@@ -334,7 +340,7 @@ function toCrawleeOptions(options, env, placeholderOptions) {
|
|
|
334
340
|
return crawleeOptions;
|
|
335
341
|
}
|
|
336
342
|
function hasApifyOnlyOptions(options) {
|
|
337
|
-
const groups = options
|
|
343
|
+
const groups = platformGroups(options);
|
|
338
344
|
const { countryCode, subdivisionCode } = resolveCountryOptions(options);
|
|
339
345
|
return !!(groups?.length || countryCode || subdivisionCode || options.password);
|
|
340
346
|
}
|
|
@@ -382,10 +388,10 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
382
388
|
// values throw at construction, not on the first newUrl() call).
|
|
383
389
|
resolveCountryOptions(options);
|
|
384
390
|
if (hasRealCustomProxyOptions(options) && hasApifyOnlyOptions(options)) {
|
|
385
|
-
throw new Error('Cannot combine custom proxies with
|
|
391
|
+
throw new Error('Cannot combine custom proxies with Scrapely Proxy! ' +
|
|
386
392
|
'It is not allowed to set "options.proxyUrls" or "options.newUrlFunction" combined with ' +
|
|
387
|
-
'"options.groups" or "options.
|
|
388
|
-
'"options.
|
|
393
|
+
'"options.groups" or "options.scrapelyProxyGroups" and "options.countryCode" or ' +
|
|
394
|
+
'"options.scrapelyProxyCountry", "options.subdivisionCode" or "options.scrapelyProxySubdivision".');
|
|
389
395
|
}
|
|
390
396
|
if (needsPlatformProxyUrls(options) && !platformEnv) {
|
|
391
397
|
throw new Error('[Scrapely] The platform proxy was requested (via the "auto" sentinel, tieredProxyConfig tiers without ' +
|
|
@@ -445,9 +451,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
445
451
|
const sid = sessionId != null ? validateProxySessionId(String(sessionId)) : undefined;
|
|
446
452
|
const info = await super.newProxyInfo(sid, options);
|
|
447
453
|
if (info && this.platformEnv && info.hostname === this.platformEnv.hostname) {
|
|
448
|
-
const groups = this.platformOptions
|
|
449
|
-
? this.platformOptions.groups
|
|
450
|
-
: this.platformOptions.apifyProxyGroups;
|
|
454
|
+
const groups = platformGroups(this.platformOptions);
|
|
451
455
|
const { countryCode, subdivisionCode } = resolveCountryOptions(this.platformOptions);
|
|
452
456
|
return {
|
|
453
457
|
...info,
|
|
@@ -483,6 +487,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
483
487
|
connectionError: 'Platform proxy authentication failed (HTTP 407). Check the injected ' +
|
|
484
488
|
'SCRAPELY_PROXY_*/APIFY_PROXY_* credentials.',
|
|
485
489
|
isManInTheMiddle: false,
|
|
490
|
+
fromPayload: false,
|
|
486
491
|
};
|
|
487
492
|
}
|
|
488
493
|
if (statusCode >= 590 && statusCode <= 599) {
|
|
@@ -491,6 +496,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
491
496
|
connected: false,
|
|
492
497
|
connectionError: `Platform proxy upstream error (HTTP ${statusCode}: ${description}).`,
|
|
493
498
|
isManInTheMiddle: false,
|
|
499
|
+
fromPayload: false,
|
|
494
500
|
};
|
|
495
501
|
}
|
|
496
502
|
if (statusCode >= 200 && statusCode < 300) {
|
|
@@ -500,6 +506,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
500
506
|
connected: parsed.connected === true,
|
|
501
507
|
connectionError: parsed.connectionError ?? '',
|
|
502
508
|
isManInTheMiddle: parsed.isManInTheMiddle === true,
|
|
509
|
+
fromPayload: true,
|
|
503
510
|
};
|
|
504
511
|
}
|
|
505
512
|
catch {
|
|
@@ -516,7 +523,10 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
516
523
|
/**
|
|
517
524
|
* Verifies the proxy configuration. For platform proxy mode with an
|
|
518
525
|
* injected status URL, performs the Apify-format access check through the
|
|
519
|
-
* gateway (`GET {statusUrl}/?format=json
|
|
526
|
+
* gateway (`GET {statusUrl}/?format=json`). When the gateway answers
|
|
527
|
+
* `connected: false` (e.g. an unknown proxy group), it throws with the
|
|
528
|
+
* gateway's `connectionError` on the platform and warns locally, like
|
|
529
|
+
* Apify's `_checkAccess`; timeouts and HTTP errors only warn.
|
|
520
530
|
* BYOP mode only verifies that a test URL can be generated.
|
|
521
531
|
*/
|
|
522
532
|
async initialize(options = {}) {
|
|
@@ -526,7 +536,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
526
536
|
}
|
|
527
537
|
// Apify-parity access check: GET {statusUrl}/?format=json *through the
|
|
528
538
|
// platform gateway* so the proxy handshake itself authenticates the
|
|
529
|
-
// run's credentials.
|
|
539
|
+
// run's credentials.
|
|
530
540
|
if (checkAccess && this.platformEnv?.statusUrl) {
|
|
531
541
|
const status = await this.fetchProxyStatus();
|
|
532
542
|
if (!status) {
|
|
@@ -535,6 +545,9 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
|
|
|
535
545
|
}
|
|
536
546
|
else {
|
|
537
547
|
this.isManInTheMiddle = status.isManInTheMiddle;
|
|
548
|
+
if (!status.connected && status.fromPayload && this.config.get('isAtHome')) {
|
|
549
|
+
throw new Error(status.connectionError);
|
|
550
|
+
}
|
|
538
551
|
if (!status.connected) {
|
|
539
552
|
console.warn(`[Scrapely] Platform proxy access check failed: ${status.connectionError}`);
|
|
540
553
|
}
|