scrapely 1.2.19 → 1.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +20 -0
- package/dist/actor.js +31 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +17 -2
- package/dist/solver.d.ts +122 -0
- package/dist/solver.js +279 -0
- package/package.json +3 -2
package/dist/actor.d.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { type ApiStorageConfig } from './storage/api-storage-client.js';
|
|
|
21
21
|
import { ChargingManager, type ChargeOptions as ChargingChargeOptions, type ChargeResult as ChargingChargeResult } from './charging.js';
|
|
22
22
|
import { createProxyConfiguration, type CreateProxyConfigurationOptions } from './proxy-configuration.js';
|
|
23
23
|
import { type BrowserConfiguration, type BrowserConfigurationOptions } from './browser-configuration.js';
|
|
24
|
+
import { type Clearance, type SolveOptions, type SolverVendorStatus } from './solver.js';
|
|
24
25
|
import type { Dataset, KeyValueStore, RequestQueue } from 'crawlee';
|
|
25
26
|
/** User function passed to {@link Actor.main} (Apify-compatible). */
|
|
26
27
|
export type UserFunc<T = unknown> = () => T | Promise<T>;
|
|
@@ -530,6 +531,25 @@ export declare class Actor {
|
|
|
530
531
|
* @returns The updated ActorRun object, or an empty object if running locally
|
|
531
532
|
*/
|
|
532
533
|
static setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<ActorRun>;
|
|
534
|
+
/**
|
|
535
|
+
* Anti-bot solver status for a vendor (free, cached for 15 s, never throws).
|
|
536
|
+
* `usable: false` means the vendor cannot be solved right now: take your
|
|
537
|
+
* fallback (for example the hosted browser), or fail the items that need a
|
|
538
|
+
* solve at once and call `setSolverDegradedStatus()`.
|
|
539
|
+
*/
|
|
540
|
+
static solverStatus(vendor: string): Promise<SolverVendorStatus>;
|
|
541
|
+
/**
|
|
542
|
+
* Solves one anti-bot challenge on the Actor's own session through the
|
|
543
|
+
* platform solver. See `solve` in `scrapely` (solver.ts) for the options.
|
|
544
|
+
*/
|
|
545
|
+
static solve(options: SolveOptions): Promise<Clearance>;
|
|
546
|
+
/**
|
|
547
|
+
* Sets the standard status message for a run whose site protection cannot
|
|
548
|
+
* be solved right now ("Site protection temporarily unavailable, results
|
|
549
|
+
* may be incomplete", plus "(N of M items skipped)" when both counts are
|
|
550
|
+
* given). Never throws: the run goes on.
|
|
551
|
+
*/
|
|
552
|
+
static setSolverDegradedStatus(skipped?: number, total?: number): Promise<void>;
|
|
533
553
|
/**
|
|
534
554
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
535
555
|
* of the ProxyConfiguration class that is already initialized.
|
package/dist/actor.js
CHANGED
|
@@ -33,6 +33,7 @@ const charging_client_js_1 = require("./charging_client.js");
|
|
|
33
33
|
const proxy_configuration_js_1 = require("./proxy-configuration.js");
|
|
34
34
|
const browser_configuration_js_1 = require("./browser-configuration.js");
|
|
35
35
|
const utils_js_1 = require("./utils.js");
|
|
36
|
+
const solver_js_1 = require("./solver.js");
|
|
36
37
|
/** "429 Run API rate limit exceeded ..." for an API error, the message otherwise. */
|
|
37
38
|
function describeRunApiError(err) {
|
|
38
39
|
const status = err.statusCode;
|
|
@@ -1073,6 +1074,36 @@ class Actor {
|
|
|
1073
1074
|
});
|
|
1074
1075
|
return (body?.data ?? {});
|
|
1075
1076
|
}
|
|
1077
|
+
/**
|
|
1078
|
+
* Anti-bot solver status for a vendor (free, cached for 15 s, never throws).
|
|
1079
|
+
* `usable: false` means the vendor cannot be solved right now: take your
|
|
1080
|
+
* fallback (for example the hosted browser), or fail the items that need a
|
|
1081
|
+
* solve at once and call `setSolverDegradedStatus()`.
|
|
1082
|
+
*/
|
|
1083
|
+
static async solverStatus(vendor) {
|
|
1084
|
+
return (0, solver_js_1.solverStatus)(vendor);
|
|
1085
|
+
}
|
|
1086
|
+
/**
|
|
1087
|
+
* Solves one anti-bot challenge on the Actor's own session through the
|
|
1088
|
+
* platform solver. See `solve` in `scrapely` (solver.ts) for the options.
|
|
1089
|
+
*/
|
|
1090
|
+
static async solve(options) {
|
|
1091
|
+
return (0, solver_js_1.solve)(options);
|
|
1092
|
+
}
|
|
1093
|
+
/**
|
|
1094
|
+
* Sets the standard status message for a run whose site protection cannot
|
|
1095
|
+
* be solved right now ("Site protection temporarily unavailable, results
|
|
1096
|
+
* may be incomplete", plus "(N of M items skipped)" when both counts are
|
|
1097
|
+
* given). Never throws: the run goes on.
|
|
1098
|
+
*/
|
|
1099
|
+
static async setSolverDegradedStatus(skipped, total) {
|
|
1100
|
+
try {
|
|
1101
|
+
await Actor.setStatusMessage((0, solver_js_1.degradedStatusMessage)(skipped, total));
|
|
1102
|
+
}
|
|
1103
|
+
catch (err) {
|
|
1104
|
+
console.warn(`Could not set the solver status message: ${err.message}`);
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1076
1107
|
/**
|
|
1077
1108
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
1078
1109
|
* of the ProxyConfiguration class that is already initialized.
|
package/dist/index.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ export { Actor, ACTOR_PERMISSION_LEVEL, type ActorInput, type InitOptions, type
|
|
|
4
4
|
export type { StorageAlias, StorageId, StorageName, StorageIdentifier, StorageIdentifierWithoutAlias, } from './storage-identifiers.js';
|
|
5
5
|
export { ApifyClient, type ApifyClientOptions, type ActorRun } from 'scrapely-client';
|
|
6
6
|
export { Configuration, type ConfigurationOptions } from './configuration.js';
|
|
7
|
+
export { DEGRADED_STATUS_MESSAGE, IpBanned, SolveAttemptError, SolverAuthError, SolverRateLimited, VendorUnavailable, degradedStatusMessage, detectVendor, isBlocked, reportSolveOutcome, solve, solverConfigured, solverStatus, stepAllowed, type Clearance, type SolveCookie, type SolveOptions, type SolverStepRequest, type SolverVendorStatus, } from './solver.js';
|
|
7
8
|
export { ScrapelyEventManager as PlatformEventManager } from './events/scrapely-event-manager.js';
|
|
8
9
|
export { ChargingManager, ChargeUncertainError, ChargeRejectedError, DEFAULT_DATASET_ITEM_EVENT, APIFY_DEFAULT_DATASET_ITEM_EVENT, SCRAPELY_DEFAULT_DATASET_ITEM_EVENT, hasDefaultDatasetItemEventInPricing, isDefaultDatasetItemEvent, resolveDefaultDatasetItemEventName, mergeChargeResults, pushDataAndCharge, type ChargeOptions, type ChargeResult, type ActorPricingInfo, } from './charging.js';
|
|
9
10
|
export { createChargingClient, pushDataChargingContext, USES_PUSH_DATA_INTERCEPTION, type PpeAwarePushDataContext, } from './charging_client.js';
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.puppeteerUtils = exports.playwrightUtils = void 0;
|
|
3
|
+
exports.LogLevel = exports.Log = exports.log = exports.RequestQueue = exports.KeyValueStore = exports.Dataset = exports.createBrowserConfiguration = exports.BrowserConfiguration = exports.getPlatformProxyEnv = exports.createProxyConfiguration = exports.validateProxySessionId = exports.isPlatformProxySentinel = exports.ProxyConfiguration = exports.ApiStorageClientFactory = exports.BillingTracker = exports.USES_PUSH_DATA_INTERCEPTION = exports.pushDataChargingContext = exports.createChargingClient = exports.pushDataAndCharge = exports.mergeChargeResults = exports.resolveDefaultDatasetItemEventName = exports.isDefaultDatasetItemEvent = exports.hasDefaultDatasetItemEventInPricing = exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = exports.APIFY_DEFAULT_DATASET_ITEM_EVENT = exports.DEFAULT_DATASET_ITEM_EVENT = exports.ChargeRejectedError = exports.ChargeUncertainError = exports.ChargingManager = exports.PlatformEventManager = exports.stepAllowed = exports.solverStatus = exports.solverConfigured = exports.solve = exports.reportSolveOutcome = exports.isBlocked = exports.detectVendor = exports.degradedStatusMessage = exports.VendorUnavailable = exports.SolverRateLimited = exports.SolverAuthError = exports.SolveAttemptError = exports.IpBanned = exports.DEGRADED_STATUS_MESSAGE = exports.Configuration = exports.ApifyClient = exports.EXIT_CODES = exports.ACTOR_PERMISSION_LEVEL = exports.Actor = exports.setCdpUrl = void 0;
|
|
4
|
+
exports.puppeteerUtils = exports.playwrightUtils = exports.LinkeDOMCrawler = exports.JSDOMCrawler = exports.BrowserCrawler = exports.BasicCrawler = exports.HttpCrawler = exports.CheerioCrawler = exports.PuppeteerCrawler = exports.PlaywrightCrawler = exports.ScrapelyRequestList = exports.enqueueLinks = exports.utils = exports.LoggerText = exports.LoggerJson = exports.Logger = void 0;
|
|
5
5
|
require("./log-env.js");
|
|
6
6
|
var patch_browser_pool_js_1 = require("./patch-browser-pool.js");
|
|
7
7
|
Object.defineProperty(exports, "setCdpUrl", { enumerable: true, get: function () { return patch_browser_pool_js_1.setCdpUrl; } });
|
|
@@ -14,6 +14,21 @@ var scrapely_client_1 = require("scrapely-client");
|
|
|
14
14
|
Object.defineProperty(exports, "ApifyClient", { enumerable: true, get: function () { return scrapely_client_1.ApifyClient; } });
|
|
15
15
|
var configuration_js_1 = require("./configuration.js");
|
|
16
16
|
Object.defineProperty(exports, "Configuration", { enumerable: true, get: function () { return configuration_js_1.Configuration; } });
|
|
17
|
+
var solver_js_1 = require("./solver.js");
|
|
18
|
+
Object.defineProperty(exports, "DEGRADED_STATUS_MESSAGE", { enumerable: true, get: function () { return solver_js_1.DEGRADED_STATUS_MESSAGE; } });
|
|
19
|
+
Object.defineProperty(exports, "IpBanned", { enumerable: true, get: function () { return solver_js_1.IpBanned; } });
|
|
20
|
+
Object.defineProperty(exports, "SolveAttemptError", { enumerable: true, get: function () { return solver_js_1.SolveAttemptError; } });
|
|
21
|
+
Object.defineProperty(exports, "SolverAuthError", { enumerable: true, get: function () { return solver_js_1.SolverAuthError; } });
|
|
22
|
+
Object.defineProperty(exports, "SolverRateLimited", { enumerable: true, get: function () { return solver_js_1.SolverRateLimited; } });
|
|
23
|
+
Object.defineProperty(exports, "VendorUnavailable", { enumerable: true, get: function () { return solver_js_1.VendorUnavailable; } });
|
|
24
|
+
Object.defineProperty(exports, "degradedStatusMessage", { enumerable: true, get: function () { return solver_js_1.degradedStatusMessage; } });
|
|
25
|
+
Object.defineProperty(exports, "detectVendor", { enumerable: true, get: function () { return solver_js_1.detectVendor; } });
|
|
26
|
+
Object.defineProperty(exports, "isBlocked", { enumerable: true, get: function () { return solver_js_1.isBlocked; } });
|
|
27
|
+
Object.defineProperty(exports, "reportSolveOutcome", { enumerable: true, get: function () { return solver_js_1.reportSolveOutcome; } });
|
|
28
|
+
Object.defineProperty(exports, "solve", { enumerable: true, get: function () { return solver_js_1.solve; } });
|
|
29
|
+
Object.defineProperty(exports, "solverConfigured", { enumerable: true, get: function () { return solver_js_1.solverConfigured; } });
|
|
30
|
+
Object.defineProperty(exports, "solverStatus", { enumerable: true, get: function () { return solver_js_1.solverStatus; } });
|
|
31
|
+
Object.defineProperty(exports, "stepAllowed", { enumerable: true, get: function () { return solver_js_1.stepAllowed; } });
|
|
17
32
|
var scrapely_event_manager_js_1 = require("./events/scrapely-event-manager.js");
|
|
18
33
|
Object.defineProperty(exports, "PlatformEventManager", { enumerable: true, get: function () { return scrapely_event_manager_js_1.ScrapelyEventManager; } });
|
|
19
34
|
var charging_js_1 = require("./charging.js");
|
package/dist/solver.d.ts
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anti-bot solver helpers (JS). Python parity: scrapely-python `scrapely.solver`.
|
|
3
|
+
*
|
|
4
|
+
* The platform runs the challenge solver behind /v2/solver; Actors never
|
|
5
|
+
* hold a solver key. Runs of Actors that may use it get SCRAPELY_SOLVER_URL
|
|
6
|
+
* and SCRAPELY_SOLVER_API_KEY (the run's own token).
|
|
7
|
+
*
|
|
8
|
+
* const st = await Actor.solverStatus('datadome'); // free, cached 15 s
|
|
9
|
+
* if (!st.usable) { ...fallback or fail the item fast... }
|
|
10
|
+
*
|
|
11
|
+
* const clearance = await Actor.solve({
|
|
12
|
+
* url, response: { status, body }, // the blocked response
|
|
13
|
+
* fetch: (req) => mySession(req), // send step requests on YOUR session
|
|
14
|
+
* sessionId: 's-42', identity: 'chrome-146-macos',
|
|
15
|
+
* });
|
|
16
|
+
* // clearance.cookies -> put them in your cookie jar (fetch already sent the rest)
|
|
17
|
+
*
|
|
18
|
+
* Errors: SolveAttemptError (retry on a fresh exit), IpBanned, VendorUnavailable
|
|
19
|
+
* (fail fast, `code`), SolverRateLimited, SolverAuthError.
|
|
20
|
+
*/
|
|
21
|
+
export declare const DEGRADED_STATUS_MESSAGE = "Site protection temporarily unavailable, results may be incomplete";
|
|
22
|
+
export declare class SolveAttemptError extends Error {
|
|
23
|
+
constructor(message: string);
|
|
24
|
+
}
|
|
25
|
+
export declare class IpBanned extends SolveAttemptError {
|
|
26
|
+
constructor(message: string);
|
|
27
|
+
}
|
|
28
|
+
export declare class SolverRateLimited extends SolveAttemptError {
|
|
29
|
+
readonly retryAfterSecs: number | null;
|
|
30
|
+
constructor(retryAfterSecs?: number | null);
|
|
31
|
+
}
|
|
32
|
+
export declare class SolverAuthError extends SolveAttemptError {
|
|
33
|
+
constructor(message: string);
|
|
34
|
+
}
|
|
35
|
+
/** The platform says this vendor cannot be solved right now. Fail fast; do not retry. */
|
|
36
|
+
export declare class VendorUnavailable extends Error {
|
|
37
|
+
readonly code: string;
|
|
38
|
+
readonly vendor: string | null;
|
|
39
|
+
readonly retryAfterSecs: number | null;
|
|
40
|
+
constructor(code: string, vendor?: string | null, retryAfterSecs?: number | null, message?: string);
|
|
41
|
+
}
|
|
42
|
+
export interface SolverVendorStatus {
|
|
43
|
+
status: 'healthy' | 'degraded' | 'down' | 'unsupported' | 'unknown';
|
|
44
|
+
usable: boolean;
|
|
45
|
+
code: string | null;
|
|
46
|
+
successRate30m: number | null;
|
|
47
|
+
since: string | null;
|
|
48
|
+
recheckAfterSecs: number;
|
|
49
|
+
}
|
|
50
|
+
export interface SolverStepRequest {
|
|
51
|
+
id: string;
|
|
52
|
+
method: 'GET' | 'POST';
|
|
53
|
+
url: string;
|
|
54
|
+
headers: Record<string, string>;
|
|
55
|
+
body?: string;
|
|
56
|
+
}
|
|
57
|
+
export interface SolveCookie {
|
|
58
|
+
name: string;
|
|
59
|
+
value: string;
|
|
60
|
+
domain: string;
|
|
61
|
+
path: string;
|
|
62
|
+
}
|
|
63
|
+
export interface SolveOptions {
|
|
64
|
+
/** The page that was blocked. */
|
|
65
|
+
url: string;
|
|
66
|
+
/** The blocked response. */
|
|
67
|
+
response: {
|
|
68
|
+
status: number;
|
|
69
|
+
body: string;
|
|
70
|
+
};
|
|
71
|
+
/** Sends one step request on the Actor's own session (same exit IP); returns its status and body. */
|
|
72
|
+
fetch: (request: SolverStepRequest) => Promise<{
|
|
73
|
+
status: number;
|
|
74
|
+
body: string;
|
|
75
|
+
}>;
|
|
76
|
+
/** 'datadome', 'akamai-sbsd' or 'auto' (default). */
|
|
77
|
+
vendor?: string;
|
|
78
|
+
/** Proxy session id (enables the platform's clearance cache for that exit). */
|
|
79
|
+
sessionId?: string;
|
|
80
|
+
/** Browser profile the TLS impersonation claims (default chrome-146-macos), or an object with locale. */
|
|
81
|
+
identity?: string | {
|
|
82
|
+
profile?: string;
|
|
83
|
+
acceptLanguage?: string;
|
|
84
|
+
timezone?: string;
|
|
85
|
+
languages?: string;
|
|
86
|
+
};
|
|
87
|
+
/** Cookie header the session holds for the site (SBSD needs it). */
|
|
88
|
+
cookieHeader?: () => string | null | Promise<string | null>;
|
|
89
|
+
/** Verifies the clearance (cookies already applied by you or by `fetch`); reported to the platform. */
|
|
90
|
+
verify?: (cookies: SolveCookie[]) => Promise<boolean> | boolean;
|
|
91
|
+
timeoutMs?: number;
|
|
92
|
+
}
|
|
93
|
+
export interface Clearance {
|
|
94
|
+
solveId: string | null;
|
|
95
|
+
cookies: SolveCookie[];
|
|
96
|
+
ttlHintSecs: number;
|
|
97
|
+
cached: boolean;
|
|
98
|
+
verified: boolean | null;
|
|
99
|
+
}
|
|
100
|
+
/** True when the platform solver API is available to this run. */
|
|
101
|
+
export declare function solverConfigured(): boolean;
|
|
102
|
+
/**
|
|
103
|
+
* The protection behind a *challenge* response. Ordinary pages that merely load a
|
|
104
|
+
* vendor script are not challenges: DataDome, PerimeterX and Kasada need a 4xx
|
|
105
|
+
* status; an SBSD bundle counts only on a short interstitial page.
|
|
106
|
+
*/
|
|
107
|
+
export declare function detectVendor(status: number, body: string): string | null;
|
|
108
|
+
/** A step may only go to the target page's own host or to captcha-delivery.com, over https. */
|
|
109
|
+
export declare function stepAllowed(stepUrl: string, targetUrl: string): boolean;
|
|
110
|
+
export declare function isBlocked(response: {
|
|
111
|
+
status: number;
|
|
112
|
+
body: string;
|
|
113
|
+
}, vendor?: string): boolean;
|
|
114
|
+
export declare function degradedStatusMessage(skipped?: number, total?: number): string;
|
|
115
|
+
/** Vendor status from the platform (free, cached for 15 s). Never throws. */
|
|
116
|
+
export declare function solverStatus(vendor: string): Promise<SolverVendorStatus>;
|
|
117
|
+
/** Solves one challenge on the Actor's session through the platform. */
|
|
118
|
+
export declare function solve(options: SolveOptions): Promise<Clearance>;
|
|
119
|
+
/** Reports your own verification of a clearance (when `verify` was not passed to solve). */
|
|
120
|
+
export declare function reportSolveOutcome(clearance: Clearance, verified: boolean): Promise<void>;
|
|
121
|
+
/** Test hook. */
|
|
122
|
+
export declare function clearSolverStatusCache(): void;
|
package/dist/solver.js
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Anti-bot solver helpers (JS). Python parity: scrapely-python `scrapely.solver`.
|
|
4
|
+
*
|
|
5
|
+
* The platform runs the challenge solver behind /v2/solver; Actors never
|
|
6
|
+
* hold a solver key. Runs of Actors that may use it get SCRAPELY_SOLVER_URL
|
|
7
|
+
* and SCRAPELY_SOLVER_API_KEY (the run's own token).
|
|
8
|
+
*
|
|
9
|
+
* const st = await Actor.solverStatus('datadome'); // free, cached 15 s
|
|
10
|
+
* if (!st.usable) { ...fallback or fail the item fast... }
|
|
11
|
+
*
|
|
12
|
+
* const clearance = await Actor.solve({
|
|
13
|
+
* url, response: { status, body }, // the blocked response
|
|
14
|
+
* fetch: (req) => mySession(req), // send step requests on YOUR session
|
|
15
|
+
* sessionId: 's-42', identity: 'chrome-146-macos',
|
|
16
|
+
* });
|
|
17
|
+
* // clearance.cookies -> put them in your cookie jar (fetch already sent the rest)
|
|
18
|
+
*
|
|
19
|
+
* Errors: SolveAttemptError (retry on a fresh exit), IpBanned, VendorUnavailable
|
|
20
|
+
* (fail fast, `code`), SolverRateLimited, SolverAuthError.
|
|
21
|
+
*/
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
exports.VendorUnavailable = exports.SolverAuthError = exports.SolverRateLimited = exports.IpBanned = exports.SolveAttemptError = exports.DEGRADED_STATUS_MESSAGE = void 0;
|
|
24
|
+
exports.solverConfigured = solverConfigured;
|
|
25
|
+
exports.detectVendor = detectVendor;
|
|
26
|
+
exports.stepAllowed = stepAllowed;
|
|
27
|
+
exports.isBlocked = isBlocked;
|
|
28
|
+
exports.degradedStatusMessage = degradedStatusMessage;
|
|
29
|
+
exports.solverStatus = solverStatus;
|
|
30
|
+
exports.solve = solve;
|
|
31
|
+
exports.reportSolveOutcome = reportSolveOutcome;
|
|
32
|
+
exports.clearSolverStatusCache = clearSolverStatusCache;
|
|
33
|
+
exports.DEGRADED_STATUS_MESSAGE = 'Site protection temporarily unavailable, results may be incomplete';
|
|
34
|
+
const COMPAT_SUFFIX = '/compat/xhrdev';
|
|
35
|
+
const STATUS_TTL_MS = 15_000;
|
|
36
|
+
const MAX_BODY = 4 * 1024 * 1024;
|
|
37
|
+
const UNAVAILABLE_CODES = new Set(['vendor_degraded', 'vendor_down', 'provider_unavailable', 'vendor_unsupported', 'solver_not_enabled']);
|
|
38
|
+
class SolveAttemptError extends Error {
|
|
39
|
+
constructor(message) {
|
|
40
|
+
super(message);
|
|
41
|
+
this.name = 'SolveAttemptError';
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
exports.SolveAttemptError = SolveAttemptError;
|
|
45
|
+
class IpBanned extends SolveAttemptError {
|
|
46
|
+
constructor(message) {
|
|
47
|
+
super(message);
|
|
48
|
+
this.name = 'IpBanned';
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
exports.IpBanned = IpBanned;
|
|
52
|
+
class SolverRateLimited extends SolveAttemptError {
|
|
53
|
+
retryAfterSecs;
|
|
54
|
+
constructor(retryAfterSecs = null) {
|
|
55
|
+
super(`solver rate limit hit (HTTP 429${retryAfterSecs !== null ? `, retry after ${retryAfterSecs}s` : ''})`);
|
|
56
|
+
this.retryAfterSecs = retryAfterSecs;
|
|
57
|
+
this.name = 'SolverRateLimited';
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
exports.SolverRateLimited = SolverRateLimited;
|
|
61
|
+
class SolverAuthError extends SolveAttemptError {
|
|
62
|
+
constructor(message) {
|
|
63
|
+
super(message);
|
|
64
|
+
this.name = 'SolverAuthError';
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
exports.SolverAuthError = SolverAuthError;
|
|
68
|
+
/** The platform says this vendor cannot be solved right now. Fail fast; do not retry. */
|
|
69
|
+
class VendorUnavailable extends Error {
|
|
70
|
+
code;
|
|
71
|
+
vendor;
|
|
72
|
+
retryAfterSecs;
|
|
73
|
+
constructor(code, vendor = null, retryAfterSecs = null, message) {
|
|
74
|
+
super(message ?? `anti-bot solver unavailable (${code}${vendor ? `, ${vendor}` : ''})`);
|
|
75
|
+
this.code = code;
|
|
76
|
+
this.vendor = vendor;
|
|
77
|
+
this.retryAfterSecs = retryAfterSecs;
|
|
78
|
+
this.name = 'VendorUnavailable';
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
exports.VendorUnavailable = VendorUnavailable;
|
|
82
|
+
function solverEnv() {
|
|
83
|
+
const url = (process.env.SCRAPELY_SOLVER_URL ?? '').trim().replace(/\/+$/, '');
|
|
84
|
+
const key = (process.env.SCRAPELY_SOLVER_API_KEY ?? '').trim();
|
|
85
|
+
return url && key ? { url, key } : null;
|
|
86
|
+
}
|
|
87
|
+
function gateway() {
|
|
88
|
+
const env = solverEnv();
|
|
89
|
+
if (!env || !env.url.endsWith(COMPAT_SUFFIX))
|
|
90
|
+
return null;
|
|
91
|
+
return { base: env.url.slice(0, -COMPAT_SUFFIX.length), key: env.key };
|
|
92
|
+
}
|
|
93
|
+
/** True when the platform solver API is available to this run. */
|
|
94
|
+
function solverConfigured() {
|
|
95
|
+
return gateway() !== null;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* The protection behind a *challenge* response. Ordinary pages that merely load a
|
|
99
|
+
* vendor script are not challenges: DataDome, PerimeterX and Kasada need a 4xx
|
|
100
|
+
* status; an SBSD bundle counts only on a short interstitial page.
|
|
101
|
+
*/
|
|
102
|
+
function detectVendor(status, body) {
|
|
103
|
+
const all = body || '';
|
|
104
|
+
const sample = all.slice(0, 20000);
|
|
105
|
+
const head = sample.slice(0, 4000).toLowerCase();
|
|
106
|
+
const challenged = status >= 400;
|
|
107
|
+
if (challenged && (head.includes('captcha-delivery.com') || /var\s+dd\s*=\s*\{/.test(sample) || head.includes('datadome')))
|
|
108
|
+
return 'datadome';
|
|
109
|
+
if (all.length <= 20000 && /<script[^>]+src="([^"]+\?v=[0-9a-f-]{36}[^"]*)"/i.test(sample))
|
|
110
|
+
return 'akamai-sbsd';
|
|
111
|
+
if (challenged && /_pxappid|px-captcha|perimeterx|\/_px\d?\//i.test(sample))
|
|
112
|
+
return 'perimeterx';
|
|
113
|
+
if (challenged && /kpsdk/i.test(sample))
|
|
114
|
+
return 'kasada';
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
/** A step may only go to the target page's own host or to captcha-delivery.com, over https. */
|
|
118
|
+
function stepAllowed(stepUrl, targetUrl) {
|
|
119
|
+
try {
|
|
120
|
+
const step = new URL(stepUrl);
|
|
121
|
+
const target = new URL(targetUrl);
|
|
122
|
+
if (step.protocol !== 'https:')
|
|
123
|
+
return false;
|
|
124
|
+
const host = step.hostname.toLowerCase();
|
|
125
|
+
return host === target.hostname.toLowerCase() || host === 'captcha-delivery.com' || host.endsWith('.captcha-delivery.com');
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
function isBlocked(response, vendor) {
|
|
132
|
+
const found = detectVendor(response.status, response.body);
|
|
133
|
+
return !vendor || vendor === 'auto' ? found !== null : found === vendor;
|
|
134
|
+
}
|
|
135
|
+
function degradedStatusMessage(skipped, total) {
|
|
136
|
+
return skipped !== undefined && total !== undefined ? `${exports.DEGRADED_STATUS_MESSAGE} (${skipped} of ${total} items skipped)` : exports.DEGRADED_STATUS_MESSAGE;
|
|
137
|
+
}
|
|
138
|
+
const statusCache = new Map();
|
|
139
|
+
function unknownStatus(code = null) {
|
|
140
|
+
return { status: 'unknown', usable: code === null, code, successRate30m: null, since: null, recheckAfterSecs: 300 };
|
|
141
|
+
}
|
|
142
|
+
/** Vendor status from the platform (free, cached for 15 s). Never throws. */
|
|
143
|
+
async function solverStatus(vendor) {
|
|
144
|
+
const name = (vendor || '').trim().toLowerCase();
|
|
145
|
+
const hit = statusCache.get(name);
|
|
146
|
+
if (hit && Date.now() - hit.at < STATUS_TTL_MS)
|
|
147
|
+
return hit.value;
|
|
148
|
+
const gw = gateway();
|
|
149
|
+
if (!gw)
|
|
150
|
+
return unknownStatus(solverEnv() ? null : 'solver_not_enabled');
|
|
151
|
+
try {
|
|
152
|
+
const res = await fetch(`${gw.base}/status?vendors=${encodeURIComponent(name)}`, {
|
|
153
|
+
headers: { authorization: `Bearer ${gw.key}` },
|
|
154
|
+
signal: AbortSignal.timeout(10_000),
|
|
155
|
+
});
|
|
156
|
+
if (!res.ok)
|
|
157
|
+
return unknownStatus();
|
|
158
|
+
const data = (await res.json());
|
|
159
|
+
const value = data.vendors?.[name] ?? unknownStatus();
|
|
160
|
+
statusCache.set(name, { at: Date.now(), value });
|
|
161
|
+
return value;
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
return unknownStatus();
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
async function api(path, body, key, base, timeoutMs) {
|
|
168
|
+
let res;
|
|
169
|
+
try {
|
|
170
|
+
res = await fetch(`${base}${path}`, {
|
|
171
|
+
method: 'POST',
|
|
172
|
+
headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
|
|
173
|
+
body: JSON.stringify(body),
|
|
174
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
// Gateway unreachable or timed out: one failed try, retried on a fresh session.
|
|
179
|
+
throw new SolveAttemptError(`solver gateway unreachable: ${err.name}`);
|
|
180
|
+
}
|
|
181
|
+
let data = {};
|
|
182
|
+
try {
|
|
183
|
+
data = await res.json();
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
data = {};
|
|
187
|
+
}
|
|
188
|
+
return { status: res.status, data, retryAfter: res.headers.get('retry-after') };
|
|
189
|
+
}
|
|
190
|
+
function handle(r, vendor) {
|
|
191
|
+
const { status, data } = r;
|
|
192
|
+
if (status === 503 && UNAVAILABLE_CODES.has(String(data.code ?? '')))
|
|
193
|
+
throw new VendorUnavailable(String(data.code), data.vendor ?? vendor, data.retryAfterSecs ?? null);
|
|
194
|
+
if (status === 429)
|
|
195
|
+
throw new SolverRateLimited(r.retryAfter ? Number(r.retryAfter) : (data.retryAfterSecs ?? null));
|
|
196
|
+
if (status === 401 || status === 403) {
|
|
197
|
+
if (data.code === 'solver_not_enabled')
|
|
198
|
+
throw new VendorUnavailable('solver_not_enabled', vendor, null, 'the anti-bot solver is not enabled for this Actor');
|
|
199
|
+
throw new SolverAuthError(`solver refused the run token (HTTP ${status})`);
|
|
200
|
+
}
|
|
201
|
+
if (status >= 400)
|
|
202
|
+
throw new SolveAttemptError(`solver HTTP ${status}: ${data.message ?? ''}`.trim());
|
|
203
|
+
return data;
|
|
204
|
+
}
|
|
205
|
+
/** Solves one challenge on the Actor's session through the platform. */
|
|
206
|
+
async function solve(options) {
|
|
207
|
+
const gw = gateway();
|
|
208
|
+
const vendor = options.vendor ?? 'auto';
|
|
209
|
+
if (!gw)
|
|
210
|
+
throw new VendorUnavailable('solver_not_enabled', vendor, null, 'the platform solver is not available to this run');
|
|
211
|
+
const timeoutMs = (options.timeoutMs ?? 60_000) + 30_000;
|
|
212
|
+
const identity = typeof options.identity === 'string' ? { profile: options.identity } : (options.identity ?? { profile: 'chrome-146-macos' });
|
|
213
|
+
const cookieHeader = async () => (options.cookieHeader ? await options.cookieHeader() : null);
|
|
214
|
+
let data = handle(await api('/solve', {
|
|
215
|
+
vendor,
|
|
216
|
+
url: options.url,
|
|
217
|
+
blocked: { status: options.response.status, body: (options.response.body ?? '').slice(0, MAX_BODY) },
|
|
218
|
+
identity,
|
|
219
|
+
timeoutMs: options.timeoutMs ?? 60_000,
|
|
220
|
+
session: { id: options.sessionId, cookieHeader: await cookieHeader() },
|
|
221
|
+
}, gw.key, gw.base, timeoutMs), vendor);
|
|
222
|
+
for (let step = 0; step < 8; step++) {
|
|
223
|
+
if (data.action === 'done') {
|
|
224
|
+
const clearance = {
|
|
225
|
+
solveId: data.solveId ?? null,
|
|
226
|
+
cookies: data.cookies ?? [],
|
|
227
|
+
ttlHintSecs: Number(data.ttlHintSecs) || 0,
|
|
228
|
+
cached: !!data.cached,
|
|
229
|
+
verified: null,
|
|
230
|
+
};
|
|
231
|
+
if (options.verify) {
|
|
232
|
+
const ok = !!(await options.verify(clearance.cookies));
|
|
233
|
+
clearance.verified = ok;
|
|
234
|
+
if (clearance.solveId) {
|
|
235
|
+
await api(`/solves/${clearance.solveId}/outcome`, { verified: ok }, gw.key, gw.base, 15_000).catch(() => undefined);
|
|
236
|
+
}
|
|
237
|
+
if (!ok)
|
|
238
|
+
throw new SolveAttemptError('still blocked after the clearance was applied');
|
|
239
|
+
}
|
|
240
|
+
return clearance;
|
|
241
|
+
}
|
|
242
|
+
if (data.action === 'failed') {
|
|
243
|
+
const message = String(data.message ?? data.reason ?? 'solve failed');
|
|
244
|
+
if (data.reason === 'ip_banned')
|
|
245
|
+
throw new IpBanned(message);
|
|
246
|
+
throw new SolveAttemptError(message);
|
|
247
|
+
}
|
|
248
|
+
if (data.action !== 'fetch' && data.action !== 'submit')
|
|
249
|
+
throw new SolveAttemptError(`unexpected solver answer: ${data.action}`);
|
|
250
|
+
const results = [];
|
|
251
|
+
for (const req of (data.requests ?? [])) {
|
|
252
|
+
if (!req || !stepAllowed(String(req.url ?? ''), options.url) || (req.method !== 'GET' && req.method !== 'POST')) {
|
|
253
|
+
throw new SolveAttemptError('the solver asked for a request outside the target site; refused');
|
|
254
|
+
}
|
|
255
|
+
let r;
|
|
256
|
+
try {
|
|
257
|
+
r = await options.fetch(req);
|
|
258
|
+
}
|
|
259
|
+
catch (err) {
|
|
260
|
+
throw new SolveAttemptError(`challenge request failed: ${err.name}`);
|
|
261
|
+
}
|
|
262
|
+
results.push({ id: req.id, status: r.status, body: (r.body ?? '').slice(0, MAX_BODY) });
|
|
263
|
+
}
|
|
264
|
+
data = handle(await api('/solve', { solveId: data.solveId, results, session: { id: options.sessionId, cookieHeader: await cookieHeader() } }, gw.key, gw.base, timeoutMs), vendor);
|
|
265
|
+
}
|
|
266
|
+
throw new SolveAttemptError('the solve did not finish');
|
|
267
|
+
}
|
|
268
|
+
/** Reports your own verification of a clearance (when `verify` was not passed to solve). */
|
|
269
|
+
async function reportSolveOutcome(clearance, verified) {
|
|
270
|
+
const gw = gateway();
|
|
271
|
+
if (!gw || !clearance.solveId)
|
|
272
|
+
return;
|
|
273
|
+
clearance.verified = verified;
|
|
274
|
+
await api(`/solves/${clearance.solveId}/outcome`, { verified }, gw.key, gw.base, 15_000).catch(() => undefined);
|
|
275
|
+
}
|
|
276
|
+
/** Test hook. */
|
|
277
|
+
function clearSolverStatusCache() {
|
|
278
|
+
statusCache.clear();
|
|
279
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scrapely",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.20",
|
|
4
4
|
"description": "Scrapely SDK for building JavaScript and TypeScript Actors",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -45,7 +45,8 @@
|
|
|
45
45
|
"test:terminal-status": "npx tsx --test tests/terminal-status-message.test.ts",
|
|
46
46
|
"test:cdp-retry": "npx tsx --test tests/cdp-retry.test.ts",
|
|
47
47
|
"test:proxy-input-aliases": "npx tsx --test tests/proxy-input-aliases.test.ts",
|
|
48
|
-
"test:charge-retries": "npx tsx --test tests/charge-retries.test.ts"
|
|
48
|
+
"test:charge-retries": "npx tsx --test tests/charge-retries.test.ts",
|
|
49
|
+
"test:solver": "npx tsx --test tests/solver.test.ts"
|
|
49
50
|
},
|
|
50
51
|
"author": "SandeepDev1",
|
|
51
52
|
"license": "ISC AND Apache-2.0",
|