scrapely 1.2.7 → 1.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/actor.d.ts CHANGED
@@ -546,8 +546,8 @@ export declare class Actor {
546
546
  * });
547
547
  * ```
548
548
  *
549
- * For compatibility with existing Actor Input UI (Input Schema), this function returns
550
- * `undefined` when `{ useApifyProxy: false }` is passed.
549
+ * Pass the proxy input field (`editor: "proxy"`) as is: this function returns
550
+ * `undefined` when `{ useScrapelyProxy: false }` is passed without custom proxies.
551
551
  *
552
552
  * @param options - Proxy configuration options
553
553
  * @returns Promise resolving to ProxyConfiguration instance, or undefined
package/dist/actor.js CHANGED
@@ -1015,8 +1015,8 @@ class Actor {
1015
1015
  * });
1016
1016
  * ```
1017
1017
  *
1018
- * For compatibility with existing Actor Input UI (Input Schema), this function returns
1019
- * `undefined` when `{ useApifyProxy: false }` is passed.
1018
+ * Pass the proxy input field (`editor: "proxy"`) as is: this function returns
1019
+ * `undefined` when `{ useScrapelyProxy: false }` is passed without custom proxies.
1020
1020
  *
1021
1021
  * @param options - Proxy configuration options
1022
1022
  * @returns Promise resolving to ProxyConfiguration instance, or undefined
@@ -53,14 +53,17 @@ export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions
53
53
  /**
54
54
  * Input-schema alias for groups. Normalized to groups.
55
55
  */
56
- apifyProxyGroups?: string[];
56
+ scrapelyProxyGroups?: string[];
57
57
  /**
58
58
  * Input-schema alias for subdivisionCode. Normalized to subdivisionCode.
59
59
  */
60
- apifyProxySubdivision?: string;
60
+ scrapelyProxySubdivision?: string;
61
61
  /**
62
62
  * Input-schema alias for countryCode. Normalized to countryCode.
63
63
  */
64
+ scrapelyProxyCountry?: string;
65
+ apifyProxyGroups?: string[];
66
+ apifyProxySubdivision?: string;
64
67
  apifyProxyCountry?: string;
65
68
  /**
66
69
  * Platform proxy password. Defaults to the password injected by the
@@ -73,9 +76,10 @@ export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions
73
76
  */
74
77
  export interface CreateProxyConfigurationOptions extends ProxyConfigurationOptions {
75
78
  /**
76
- * For compatibility with existing Actor Input UI (Input Schema).
77
- * When set to `false`, this method returns `undefined`.
79
+ * The proxy input field's switch (input schema `editor: "proxy"`). When
80
+ * `false` and no custom proxies are given, this method returns `undefined`.
78
81
  */
82
+ useScrapelyProxy?: boolean;
79
83
  useApifyProxy?: boolean;
80
84
  /**
81
85
  * Whether to verify the proxy configuration by testing a connection.
@@ -39,14 +39,17 @@ function hasCustomProxyOptions(options) {
39
39
  }
40
40
  function normalizeProxyInput(options) {
41
41
  const normalized = { ...options };
42
- if (normalized.apifyProxyGroups && !normalized.groups) {
43
- normalized.groups = normalized.apifyProxyGroups;
42
+ const groups = normalized.scrapelyProxyGroups ?? normalized.apifyProxyGroups;
43
+ if (groups && !normalized.groups) {
44
+ normalized.groups = groups;
44
45
  }
45
- if (normalized.apifyProxyCountry && !normalized.countryCode) {
46
- normalized.countryCode = normalized.apifyProxyCountry;
46
+ const countryCode = normalized.scrapelyProxyCountry ?? normalized.apifyProxyCountry;
47
+ if (countryCode && !normalized.countryCode) {
48
+ normalized.countryCode = countryCode;
47
49
  }
48
- if (normalized.apifyProxySubdivision && !normalized.subdivisionCode) {
49
- normalized.subdivisionCode = normalized.apifyProxySubdivision;
50
+ const subdivisionCode = normalized.scrapelyProxySubdivision ?? normalized.apifyProxySubdivision;
51
+ if (subdivisionCode && !normalized.subdivisionCode) {
52
+ normalized.subdivisionCode = subdivisionCode;
50
53
  }
51
54
  return normalized;
52
55
  }
@@ -60,17 +63,19 @@ function normalizeProxyInput(options) {
60
63
  */
61
64
  async function createProxyConfiguration(options = {}) {
62
65
  const normalized = normalizeProxyInput(options);
63
- const { useApifyProxy, checkAccess = true, ...proxyOptions } = normalized;
64
- // Apify parity: `useApifyProxy: false` explicitly disables the PLATFORM proxy
66
+ const { useScrapelyProxy, useApifyProxy, checkAccess = true, ...proxyOptions } = normalized;
67
+ // `useScrapelyProxy: false` (or the legacy `useApifyProxy: false`; a boolean
68
+ // useScrapelyProxy wins) explicitly disables the PLATFORM proxy
65
69
  // only — custom proxies from the same input must still be used. The standard
66
- // "Custom proxy URLs" input shape is `{ useApifyProxy: false, proxyUrls: [...] }`;
70
+ // "Custom proxy URLs" input shape is `{ useScrapelyProxy: false, proxyUrls: [...] }`;
67
71
  // returning undefined for it would silently drop the user's own proxies and
68
72
  // crawl on their real IPs. We only bail when NO custom proxy mode is given —
69
73
  // proxyUrls, tieredProxyUrls, tieredProxyConfig and newUrlFunction all count
70
74
  // (Python SDK Actor.create_proxy_configuration parity).
71
- const dontUseApifyProxy = useApifyProxy === false;
75
+ const usePlatformProxy = typeof useScrapelyProxy === 'boolean' ? useScrapelyProxy : useApifyProxy;
76
+ const dontUsePlatformProxy = usePlatformProxy === false;
72
77
  const dontUseCustomProxies = !hasCustomProxyOptions(proxyOptions);
73
- if (dontUseApifyProxy && dontUseCustomProxies) {
78
+ if (dontUsePlatformProxy && dontUseCustomProxies) {
74
79
  return undefined;
75
80
  }
76
81
  if (!hasCustomProxyOptions(proxyOptions)) {
@@ -89,7 +89,10 @@ export declare class ProxyConfiguration extends CrawleeProxyConfiguration {
89
89
  /**
90
90
  * Verifies the proxy configuration. For platform proxy mode with an
91
91
  * injected status URL, performs the Apify-format access check through the
92
- * gateway (`GET {statusUrl}/?format=json`, best-effort, never fatal);
92
+ * gateway (`GET {statusUrl}/?format=json`). When the gateway answers
93
+ * `connected: false` (e.g. an unknown proxy group), it throws with the
94
+ * gateway's `connectionError` on the platform and warns locally, like
95
+ * Apify's `_checkAccess`; timeouts and HTTP errors only warn.
93
96
  * BYOP mode only verifies that a test URL can be generated.
94
97
  */
95
98
  initialize(options?: {
@@ -73,6 +73,12 @@ function validateProxySessionId(sessionId) {
73
73
  }
74
74
  return sessionId;
75
75
  }
76
+ /** `groups`, else the input-schema alias `scrapelyProxyGroups`, else the legacy `apifyProxyGroups`. */
77
+ function platformGroups(options) {
78
+ if (options.groups?.length)
79
+ return options.groups;
80
+ return options.scrapelyProxyGroups ?? options.apifyProxyGroups;
81
+ }
76
82
  /**
77
83
  * Resolves and validates the country/subdivision options (Apify parity).
78
84
  * Country must be a two-letter ISO code; subdivision an ISO 3166-2 code
@@ -80,8 +86,8 @@ function validateProxySessionId(sessionId) {
80
86
  * the normalized values used by the username grammar.
81
87
  */
82
88
  function resolveCountryOptions(options) {
83
- const countryCode = options.countryCode ?? options.apifyProxyCountry;
84
- const subdivisionCode = options.subdivisionCode ?? options.apifyProxySubdivision;
89
+ const countryCode = options.countryCode ?? options.scrapelyProxyCountry ?? options.apifyProxyCountry;
90
+ const subdivisionCode = options.subdivisionCode ?? options.scrapelyProxySubdivision ?? options.apifyProxySubdivision;
85
91
  if (countryCode != null && !COUNTRY_CODE_REGEX.test(countryCode)) {
86
92
  throw new Error(`ProxyConfiguration: "countryCode" must be a two-letter ISO country code (got "${countryCode}").`);
87
93
  }
@@ -225,7 +231,7 @@ async function requestThroughProxy(targetUrl, proxyUrl, timeoutMs) {
225
231
  * server.
226
232
  */
227
233
  function buildPlatformProxyUrl(options, env, sessionId) {
228
- const groups = options.groups?.length ? options.groups : options.apifyProxyGroups;
234
+ const groups = platformGroups(options);
229
235
  const { countryCode, subdivisionCode } = resolveCountryOptions(options);
230
236
  const password = options.password ?? env.password;
231
237
  // Apify username grammar: groups, then session, then country — with the
@@ -334,7 +340,7 @@ function toCrawleeOptions(options, env, placeholderOptions) {
334
340
  return crawleeOptions;
335
341
  }
336
342
  function hasApifyOnlyOptions(options) {
337
- const groups = options.groups?.length ? options.groups : options.apifyProxyGroups;
343
+ const groups = platformGroups(options);
338
344
  const { countryCode, subdivisionCode } = resolveCountryOptions(options);
339
345
  return !!(groups?.length || countryCode || subdivisionCode || options.password);
340
346
  }
@@ -382,10 +388,10 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
382
388
  // values throw at construction, not on the first newUrl() call).
383
389
  resolveCountryOptions(options);
384
390
  if (hasRealCustomProxyOptions(options) && hasApifyOnlyOptions(options)) {
385
- throw new Error('Cannot combine custom proxies with Apify Proxy! ' +
391
+ throw new Error('Cannot combine custom proxies with Scrapely Proxy! ' +
386
392
  'It is not allowed to set "options.proxyUrls" or "options.newUrlFunction" combined with ' +
387
- '"options.groups" or "options.apifyProxyGroups" and "options.countryCode" or ' +
388
- '"options.apifyProxyCountry", "options.subdivisionCode" or "options.apifyProxySubdivision".');
393
+ '"options.groups" or "options.scrapelyProxyGroups" and "options.countryCode" or ' +
394
+ '"options.scrapelyProxyCountry", "options.subdivisionCode" or "options.scrapelyProxySubdivision".');
389
395
  }
390
396
  if (needsPlatformProxyUrls(options) && !platformEnv) {
391
397
  throw new Error('[Scrapely] The platform proxy was requested (via the "auto" sentinel, tieredProxyConfig tiers without ' +
@@ -445,9 +451,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
445
451
  const sid = sessionId != null ? validateProxySessionId(String(sessionId)) : undefined;
446
452
  const info = await super.newProxyInfo(sid, options);
447
453
  if (info && this.platformEnv && info.hostname === this.platformEnv.hostname) {
448
- const groups = this.platformOptions.groups?.length
449
- ? this.platformOptions.groups
450
- : this.platformOptions.apifyProxyGroups;
454
+ const groups = platformGroups(this.platformOptions);
451
455
  const { countryCode, subdivisionCode } = resolveCountryOptions(this.platformOptions);
452
456
  return {
453
457
  ...info,
@@ -483,6 +487,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
483
487
  connectionError: 'Platform proxy authentication failed (HTTP 407). Check the injected ' +
484
488
  'SCRAPELY_PROXY_*/APIFY_PROXY_* credentials.',
485
489
  isManInTheMiddle: false,
490
+ fromPayload: false,
486
491
  };
487
492
  }
488
493
  if (statusCode >= 590 && statusCode <= 599) {
@@ -491,6 +496,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
491
496
  connected: false,
492
497
  connectionError: `Platform proxy upstream error (HTTP ${statusCode}: ${description}).`,
493
498
  isManInTheMiddle: false,
499
+ fromPayload: false,
494
500
  };
495
501
  }
496
502
  if (statusCode >= 200 && statusCode < 300) {
@@ -500,6 +506,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
500
506
  connected: parsed.connected === true,
501
507
  connectionError: parsed.connectionError ?? '',
502
508
  isManInTheMiddle: parsed.isManInTheMiddle === true,
509
+ fromPayload: true,
503
510
  };
504
511
  }
505
512
  catch {
@@ -516,7 +523,10 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
516
523
  /**
517
524
  * Verifies the proxy configuration. For platform proxy mode with an
518
525
  * injected status URL, performs the Apify-format access check through the
519
- * gateway (`GET {statusUrl}/?format=json`, best-effort, never fatal);
526
+ * gateway (`GET {statusUrl}/?format=json`). When the gateway answers
527
+ * `connected: false` (e.g. an unknown proxy group), it throws with the
528
+ * gateway's `connectionError` on the platform and warns locally, like
529
+ * Apify's `_checkAccess`; timeouts and HTTP errors only warn.
520
530
  * BYOP mode only verifies that a test URL can be generated.
521
531
  */
522
532
  async initialize(options = {}) {
@@ -526,7 +536,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
526
536
  }
527
537
  // Apify-parity access check: GET {statusUrl}/?format=json *through the
528
538
  // platform gateway* so the proxy handshake itself authenticates the
529
- // run's credentials. Best-effort — never hard fails the run.
539
+ // run's credentials.
530
540
  if (checkAccess && this.platformEnv?.statusUrl) {
531
541
  const status = await this.fetchProxyStatus();
532
542
  if (!status) {
@@ -535,6 +545,9 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
535
545
  }
536
546
  else {
537
547
  this.isManInTheMiddle = status.isManInTheMiddle;
548
+ if (!status.connected && status.fromPayload && this.config.get('isAtHome')) {
549
+ throw new Error(status.connectionError);
550
+ }
538
551
  if (!status.connected) {
539
552
  console.warn(`[Scrapely] Platform proxy access check failed: ${status.connectionError}`);
540
553
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "scrapely",
3
- "version": "1.2.7",
3
+ "version": "1.2.8",
4
4
  "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",