scrapely 1.2.29 → 1.2.32

package/dist/client-loader.d.ts

@@ -1,6 +1,6 @@
 /**
  * scrapely-client, loaded on first use (axios, proxy-agent and friends cost
- * ~0.7 s under gVisor, and most runs never need the API client).
+ * ~0.7 s on the platform, and most runs never need the API client).
  *
  * `USES_PUSH_DATA_INTERCEPTION` and `pushDataChargingContext` come from this
  * module (not declared in the SDK) so the SDK and the client share a single

package/dist/index.d.mts

@@ -11,6 +11,19 @@ export declare const BasicCrawler: typeof import("crawlee").BasicCrawler;
 export declare const BrowserCrawler: typeof import("crawlee").BrowserCrawler;
 export declare const JSDOMCrawler: typeof import("crawlee").JSDOMCrawler;
 export declare const LinkeDOMCrawler: typeof import("crawlee").LinkeDOMCrawler;
+export declare const AdaptivePlaywrightCrawler: typeof import("crawlee").AdaptivePlaywrightCrawler;
+export declare const RenderingTypePredictor: typeof import("crawlee").RenderingTypePredictor;
+export declare const SessionPool: typeof import("crawlee").SessionPool;
+export declare const Session: typeof import("crawlee").Session;
+export declare const AutoscaledPool: typeof import("crawlee").AutoscaledPool;
+export declare const createBasicRouter: typeof import("crawlee").createBasicRouter;
+export declare const createHttpRouter: typeof import("crawlee").createHttpRouter;
+export declare const createCheerioRouter: typeof import("crawlee").createCheerioRouter;
+export declare const createJSDOMRouter: typeof import("crawlee").createJSDOMRouter;
+export declare const createLinkeDOMRouter: typeof import("crawlee").createLinkeDOMRouter;
+export declare const createPlaywrightRouter: typeof import("crawlee").createPlaywrightRouter;
+export declare const createAdaptivePlaywrightRouter: typeof import("crawlee").createAdaptivePlaywrightRouter;
+export declare const createPuppeteerRouter: typeof import("crawlee").createPuppeteerRouter;
 export declare const ApifyClient: typeof import("scrapely-client").ScrapelyClient;
 export declare const playwrightUtils: typeof Sdk.playwrightUtils;
 export declare const puppeteerUtils: typeof Sdk.puppeteerUtils;

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 import './log-env.js';
 import type * as CrawleeBasic from '@crawlee/basic';
 import type * as CrawleeBrowser from '@crawlee/browser';
+import type * as CrawleeCore from '@crawlee/core';
 import type * as CrawleeCheerio from '@crawlee/cheerio';
 import type * as CrawleeHttp from '@crawlee/http';
 import type * as CrawleeJsdom from '@crawlee/jsdom';
@@ -27,6 +28,7 @@ export { ProxyConfiguration, isPlatformProxySentinel, validateProxySessionId } f
 export { createProxyConfiguration, getPlatformProxyEnv, type ProxyConfigurationOptions, type CreateProxyConfigurationOptions, type PlatformProxyEnv, type ProxyInfo, type ProxyConfigurationFunction, } from './proxy-configuration.js';
 export { BrowserConfiguration, createBrowserConfiguration, type BrowserConfigurationOptions, } from './browser-configuration.js';
 export { Dataset, KeyValueStore, RequestQueue, log, Log, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
+export { Request, RequestList, Router, RequestState, EventType, Statistics, Snapshotter, SystemStatus, NonRetryableError, RetryRequestError, MissingRouteError, CriticalError, useState, mergeCookies, } from '@crawlee/core';
 export type { LoggerOptions, KeyValueStoreIteratorOptions, KeyValueStoreOptions, DatasetConsumer, DatasetContent, DatasetDataOptions, DatasetIteratorOptions, DatasetMapper, DatasetOptions, DatasetReducer, QueueOperationInfo, RequestQueueOperationOptions, RequestQueueOptions, } from '@crawlee/core';
 export type { KeyConsumer } from './storage/key-value-store-client.js';
 export { utils, enqueueLinks, type EnqueueLinksOptions } from './utils.js';
@@ -47,5 +49,23 @@ export declare const JSDOMCrawler: typeof CrawleeJsdom.JSDOMCrawler;
 export type JSDOMCrawler = CrawleeJsdom.JSDOMCrawler;
 export declare const LinkeDOMCrawler: typeof CrawleeLinkedom.LinkeDOMCrawler;
 export type LinkeDOMCrawler = CrawleeLinkedom.LinkeDOMCrawler;
+export declare const SessionPool: typeof CrawleeCore.SessionPool;
+export type SessionPool = CrawleeCore.SessionPool;
+export declare const Session: typeof CrawleeCore.Session;
+export type Session = CrawleeCore.Session;
+export declare const AutoscaledPool: typeof CrawleeCore.AutoscaledPool;
+export type AutoscaledPool = CrawleeCore.AutoscaledPool;
+export declare const AdaptivePlaywrightCrawler: typeof CrawleePlaywright.AdaptivePlaywrightCrawler;
+export type AdaptivePlaywrightCrawler = CrawleePlaywright.AdaptivePlaywrightCrawler;
+export declare const RenderingTypePredictor: typeof CrawleePlaywright.RenderingTypePredictor;
+export type RenderingTypePredictor = CrawleePlaywright.RenderingTypePredictor;
+export declare const createBasicRouter: typeof CrawleeBasic.createBasicRouter;
+export declare const createHttpRouter: typeof CrawleeHttp.createHttpRouter;
+export declare const createCheerioRouter: typeof CrawleeCheerio.createCheerioRouter;
+export declare const createJSDOMRouter: typeof CrawleeJsdom.createJSDOMRouter;
+export declare const createLinkeDOMRouter: typeof CrawleeLinkedom.createLinkeDOMRouter;
+export declare const createPlaywrightRouter: typeof CrawleePlaywright.createPlaywrightRouter;
+export declare const createAdaptivePlaywrightRouter: typeof CrawleePlaywright.createAdaptivePlaywrightRouter;
+export declare const createPuppeteerRouter: typeof CrawleePuppeteer.createPuppeteerRouter;
 export declare const playwrightUtils: typeof CrawleePlaywright.playwrightUtils;
 export declare const puppeteerUtils: typeof CrawleePuppeteer.puppeteerUtils;

package/dist/index.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Logger = exports.LogLevel = exports.Log = exports.log = exports.RequestQueue = exports.KeyValueStore = exports.Dataset = exports.createBrowserConfiguration = exports.BrowserConfiguration = exports.getPlatformProxyEnv = exports.createProxyConfiguration = exports.validateProxySessionId = exports.isPlatformProxySentinel = exports.ProxyConfiguration = exports.ApiStorageClientFactory = exports.BillingTracker = exports.createChargingClient = exports.validateRowCharges = exports.pushDataWithCharges = exports.pushDataAndCharge = exports.mergeChargeResults = exports.resolveDefaultDatasetItemEventName = exports.isDefaultDatasetItemEvent = exports.hasDefaultDatasetItemEventInPricing = exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = exports.APIFY_DEFAULT_DATASET_ITEM_EVENT = exports.DEFAULT_DATASET_ITEM_EVENT = exports.ChargeRejectedError = exports.ChargeUncertainError = exports.ChargingManager = exports.PlatformEventManager = exports.stepAllowed = exports.solverStatus = exports.solverConfigured = exports.solve = exports.reportSolveOutcome = exports.isBlocked = exports.detectVendor = exports.degradedStatusMessage = exports.VendorUnavailable = exports.SolverRateLimited = exports.SolverAuthError = exports.SolveAttemptError = exports.IpBanned = exports.DEGRADED_STATUS_MESSAGE = exports.Configuration = exports.EXIT_CODES = exports.ACTOR_PERMISSION_LEVEL = exports.Actor = exports.setCdpUrl = void 0;
-exports.ScrapelyRequestList = exports.enqueueLinks = exports.utils = exports.LoggerText = exports.LoggerJson = void 0;
+exports.ScrapelyRequestList = exports.enqueueLinks = exports.utils = exports.mergeCookies = exports.useState = exports.CriticalError = exports.MissingRouteError = exports.RetryRequestError = exports.NonRetryableError = exports.SystemStatus = exports.Snapshotter = exports.Statistics = exports.RequestState = exports.Router = exports.RequestList = exports.Request = exports.LoggerText = exports.LoggerJson = void 0;
 require("./log-env.js");
 const lazy_exports_js_1 = require("./lazy-exports.js");
 var patch_browser_pool_js_1 = require("./patch-browser-pool.js");
@@ -74,6 +74,22 @@ Object.defineProperty(exports, "LogLevel", { enumerable: true, get: function ()
 Object.defineProperty(exports, "Logger", { enumerable: true, get: function () { return core_1.Logger; } });
 Object.defineProperty(exports, "LoggerJson", { enumerable: true, get: function () { return core_1.LoggerJson; } });
 Object.defineProperty(exports, "LoggerText", { enumerable: true, get: function () { return core_1.LoggerText; } });
+// Cheap core classes the reference SDK re-exports from crawlee. @crawlee/core is
+// already loaded by the exports above, so these are plain re-exports.
+var core_2 = require("@crawlee/core");
+Object.defineProperty(exports, "Request", { enumerable: true, get: function () { return core_2.Request; } });
+Object.defineProperty(exports, "RequestList", { enumerable: true, get: function () { return core_2.RequestList; } });
+Object.defineProperty(exports, "Router", { enumerable: true, get: function () { return core_2.Router; } });
+Object.defineProperty(exports, "RequestState", { enumerable: true, get: function () { return core_2.RequestState; } });
+Object.defineProperty(exports, "Statistics", { enumerable: true, get: function () { return core_2.Statistics; } });
+Object.defineProperty(exports, "Snapshotter", { enumerable: true, get: function () { return core_2.Snapshotter; } });
+Object.defineProperty(exports, "SystemStatus", { enumerable: true, get: function () { return core_2.SystemStatus; } });
+Object.defineProperty(exports, "NonRetryableError", { enumerable: true, get: function () { return core_2.NonRetryableError; } });
+Object.defineProperty(exports, "RetryRequestError", { enumerable: true, get: function () { return core_2.RetryRequestError; } });
+Object.defineProperty(exports, "MissingRouteError", { enumerable: true, get: function () { return core_2.MissingRouteError; } });
+Object.defineProperty(exports, "CriticalError", { enumerable: true, get: function () { return core_2.CriticalError; } });
+Object.defineProperty(exports, "useState", { enumerable: true, get: function () { return core_2.useState; } });
+Object.defineProperty(exports, "mergeCookies", { enumerable: true, get: function () { return core_2.mergeCookies; } });
 // Apify v2 compatible utils namespace
 var utils_js_1 = require("./utils.js");
 Object.defineProperty(exports, "utils", { enumerable: true, get: function () { return utils_js_1.utils; } });

package/dist/index.mjs

@@ -24,6 +24,19 @@ export const BasicCrawler = lazyExport('BasicCrawler', 'class');
 export const BrowserCrawler = lazyExport('BrowserCrawler', 'class');
 export const JSDOMCrawler = lazyExport('JSDOMCrawler', 'class');
 export const LinkeDOMCrawler = lazyExport('LinkeDOMCrawler', 'class');
+export const AdaptivePlaywrightCrawler = lazyExport('AdaptivePlaywrightCrawler', 'class');
+export const RenderingTypePredictor = lazyExport('RenderingTypePredictor', 'class');
+export const SessionPool = lazyExport('SessionPool', 'class');
+export const Session = lazyExport('Session', 'class');
+export const AutoscaledPool = lazyExport('AutoscaledPool', 'class');
+export const createBasicRouter = lazyExport('createBasicRouter', 'class');
+export const createHttpRouter = lazyExport('createHttpRouter', 'class');
+export const createCheerioRouter = lazyExport('createCheerioRouter', 'class');
+export const createJSDOMRouter = lazyExport('createJSDOMRouter', 'class');
+export const createLinkeDOMRouter = lazyExport('createLinkeDOMRouter', 'class');
+export const createPlaywrightRouter = lazyExport('createPlaywrightRouter', 'class');
+export const createAdaptivePlaywrightRouter = lazyExport('createAdaptivePlaywrightRouter', 'class');
+export const createPuppeteerRouter = lazyExport('createPuppeteerRouter', 'class');
 export const ApifyClient = lazyExport('ApifyClient', 'class');
 export const playwrightUtils = lazyExport('playwrightUtils', 'object');
 export const puppeteerUtils = lazyExport('puppeteerUtils', 'object');

package/dist/lazy-exports.js

@@ -7,7 +7,7 @@ exports.lazyExport = lazyExport;
 /**
  * Public SDK names that load heavy packages: Crawlee's crawlers (jsdom,
  * linkedom, cheerio, the browser pool, playwright) and scrapely-client.
- * Importing the SDK used to load all of them, ~4 s at 1 core under gVisor;
+ * Importing the SDK used to load all of them, ~4 s at 1 core on the platform;
  * they now load on first access.
  *
  * - CommonJS (`require('scrapely').PlaywrightCrawler`): a getter on the
@@ -30,6 +30,19 @@ exports.LAZY_EXPORTS = {
     BrowserCrawler: ['@crawlee/browser', 'BrowserCrawler'],
     JSDOMCrawler: ['@crawlee/jsdom', 'JSDOMCrawler'],
     LinkeDOMCrawler: ['@crawlee/linkedom', 'LinkeDOMCrawler'],
+    AdaptivePlaywrightCrawler: ['@crawlee/playwright', 'AdaptivePlaywrightCrawler'],
+    RenderingTypePredictor: ['@crawlee/playwright', 'RenderingTypePredictor'],
+    SessionPool: ['@crawlee/core', 'SessionPool'],
+    Session: ['@crawlee/core', 'Session'],
+    AutoscaledPool: ['@crawlee/core', 'AutoscaledPool'],
+    createBasicRouter: ['@crawlee/basic', 'createBasicRouter'],
+    createHttpRouter: ['@crawlee/http', 'createHttpRouter'],
+    createCheerioRouter: ['@crawlee/cheerio', 'createCheerioRouter'],
+    createJSDOMRouter: ['@crawlee/jsdom', 'createJSDOMRouter'],
+    createLinkeDOMRouter: ['@crawlee/linkedom', 'createLinkeDOMRouter'],
+    createPlaywrightRouter: ['@crawlee/playwright', 'createPlaywrightRouter'],
+    createAdaptivePlaywrightRouter: ['@crawlee/playwright', 'createAdaptivePlaywrightRouter'],
+    createPuppeteerRouter: ['@crawlee/puppeteer', 'createPuppeteerRouter'],
     playwrightUtils: ['@crawlee/playwright', 'playwrightUtils'],
     puppeteerUtils: ['@crawlee/puppeteer', 'puppeteerUtils'],
     ApifyClient: ['scrapely-client', 'ApifyClient'],

package/dist/node-wrapper.d.ts

@@ -4,7 +4,7 @@
  * The base images preload the SDK's browser patch into every Node process
  * (NODE_OPTIONS=--import /opt/scrapely/register.mjs). `npm start` is two Node
  * processes: npm, which only spawns the start script, and the actor. npm itself
- * costs ~0.6 s at 1 core and ~2.7 s at 0.25 core (1 GB run) under gVisor.
+ * costs ~0.6 s at 1 core and ~2.7 s at 0.25 core (1 GB run) on the platform.
  *
  * - npm / npx / yarn / pnpm processes never run actor code, so they get no
  *   loader hooks. Their children inherit NODE_OPTIONS and get the preload.

package/dist/patch-browser-pool.d.ts

@@ -66,7 +66,7 @@ export declare function patchLoadedCrawleeCopies(): void;
  * - `cdpUrl` becomes a valid launcher option (PlaywrightLauncher and
  *   PuppeteerLauncher copy BrowserLauncher's shape when their file loads, so
  *   each class gets it; browser-launcher.js loads first, so the copies inherit it).
- * - Local launches get gVisor-safe Chrome args under runsc. Playwright/Puppeteer
+ * - Local launches get sandbox-safe Chrome args on the platform. Playwright/Puppeteer
  *   pool launches use CDP and skip this.
  *
  * @internal Exported for tests.

package/dist/patch-browser-pool.js

@@ -60,20 +60,20 @@ function isAtHome() {
 function browserPatchDisabled() {
     return process.env.SCRAPELY_BROWSER_PATCH === '0';
 }
-/** gVisor lacks crashpad support; ReadonlyRootfs + tmpfs /tmp loses crash dirs. */
-const GVISOR_CHROME_ARGS = [
+/** Platform sandbox: no crash reporter; read-only rootfs + tmpfs /tmp loses crash dirs. */
+const SANDBOX_CHROME_ARGS = [
     '--disable-breakpad',
     '--disable-crash-reporter',
     '--disable-dev-shm-usage',
     '--crash-dumps-dir=/tmp',
 ];
-/** Add gVisor-safe Chrome args to local launch options when running under runsc. */
-function withGVisorArgs(options) {
+/** Add sandbox-safe Chrome args to local launch options when running on the platform. */
+function withSandboxArgs(options) {
     const opts = { ...(options ?? {}) };
-    if (process.env.SCRAPELY_CONTAINER_RUNTIME !== 'runsc')
+    if (!isAtHome())
         return opts;
     const args = [...(opts.args ?? [])];
-    for (const arg of GVISOR_CHROME_ARGS) {
+    for (const arg of SANDBOX_CHROME_ARGS) {
         if (!args.includes(arg))
             args.push(arg);
     }
@@ -84,7 +84,7 @@ function withGVisorArgs(options) {
  * Require CDP_URL for platform browser runs. Local launch is disabled on the
  * platform unless the build is unpatched (SCRAPELY_BROWSER_PATCH=0) — otherwise
  * a worker container without CDP_URL is a misconfiguration and must fail
- * loudly instead of launching Chromium inside the (gVisor) container.
+ * loudly instead of launching Chromium inside the platform container.
  */
 function requireCdpUrl() {
     const cdpUrl = getCdpUrl();
@@ -491,11 +491,11 @@ function patchLoadedCrawleeCopies() {
         }
     }
 }
-// ─── Crawlee launchers: `cdpUrl` option + gVisor-safe local launch args ─────
+// ─── Crawlee launchers: `cdpUrl` option + sandbox-safe local launch args ─────
 //
 // Applied when a launcher module loads (Module._load hook below), never by
 // importing Crawlee up front: loading @crawlee/browser-pool and playwright
-// eagerly cost ~2.4 s per Node process under gVisor.
+// eagerly cost ~2.4 s per Node process on the platform.
 /** Requests that load a Crawlee launcher file (`require('./internals/browser-launcher')`). */
 const LAUNCHER_REQUEST_RE = /(^|[/\\])internals[/\\](browser|playwright|puppeteer)-launcher(\.js)?$/;
 /** Cached module filenames of Crawlee launcher files (any copy). */
@@ -533,7 +533,7 @@ function addCdpUrlOption(Launcher, parentFilename) {
  * - `cdpUrl` becomes a valid launcher option (PlaywrightLauncher and
  *   PuppeteerLauncher copy BrowserLauncher's shape when their file loads, so
  *   each class gets it; browser-launcher.js loads first, so the copies inherit it).
- * - Local launches get gVisor-safe Chrome args under runsc. Playwright/Puppeteer
+ * - Local launches get sandbox-safe Chrome args on the platform. Playwright/Puppeteer
  *   pool launches use CDP and skip this.
  *
  * @internal Exported for tests.
@@ -555,17 +555,17 @@ function patchBrowserLauncherModule(mod, parentFilename) {
     }
     const BrowserLauncher = mod.BrowserLauncher;
     const proto = BrowserLauncher?.prototype;
-    if (!proto || typeof proto.createLaunchOptions !== 'function' || proto._scrapelyGVisorArgsPatched)
+    if (!proto || typeof proto.createLaunchOptions !== 'function' || proto._scrapelySandboxArgsPatched)
         return;
     const origCreateLaunchOptions = proto.createLaunchOptions;
     proto.createLaunchOptions = function (...args) {
         const launchOptions = origCreateLaunchOptions.apply(this, args);
-        // gVisor doesn't support Chrome's crashpad_handler process. Also needed
+        // The sandbox has no Chrome crashpad_handler support. Also needed
         // with ReadonlyRootfs + tmpfs /tmp where /tmp/.X11-unix and the crashpad
         // database dir don't persist from the base image.
-        if (process.env.SCRAPELY_CONTAINER_RUNTIME === 'runsc' && launchOptions) {
+        if (isAtHome() && launchOptions) {
             launchOptions.args = launchOptions.args || [];
-            for (const arg of GVISOR_CHROME_ARGS) {
+            for (const arg of SANDBOX_CHROME_ARGS) {
                 if (!launchOptions.args.includes(arg)) {
                     launchOptions.args.push(arg);
                 }
@@ -573,7 +573,7 @@ function patchBrowserLauncherModule(mod, parentFilename) {
         }
         return launchOptions;
     };
-    proto._scrapelyGVisorArgsPatched = true;
+    proto._scrapelySandboxArgsPatched = true;
 }
 /**
  * Split our `browserConfiguration` option off raw launch options. The options
@@ -655,7 +655,7 @@ function patchRawBrowserType(chromium, browserType, name) {
     browserType.launch = async function rawLaunchPatched(launchOptions = {}) {
         const { options, browserConfiguration } = takeBrowserConfiguration(launchOptions);
         if (browserPatchDisabled())
-            return origLaunch.call(this, withGVisorArgs(options));
+            return origLaunch.call(this, withSandboxArgs(options));
         const cdpUrl = rawLaunchCdpUrl(options, browserConfiguration);
         if (!cdpUrl) {
             if (!isAtHome())
@@ -674,7 +674,7 @@ function patchRawBrowserType(chromium, browserType, name) {
             const { options, browserConfiguration } = takeBrowserConfiguration(lpcArgs[optionsIndex]);
             const withOptions = (opts) => optionsIndex === 1 ? [userDataDir, opts] : [opts];
             if (browserPatchDisabled())
-                return origLpc.call(this, ...withOptions(withGVisorArgs(options)));
+                return origLpc.call(this, ...withOptions(withSandboxArgs(options)));
             const cdpUrl = rawLaunchCdpUrl(options, browserConfiguration);
             if (!cdpUrl) {
                 if (!isAtHome()) {
@@ -719,7 +719,7 @@ function patchRawPlaywrightModule(pw) {
 async function puppeteerLaunchPatched(self, origLaunch, moduleName, launchOptions = {}) {
     const { options, browserConfiguration } = takeBrowserConfiguration(launchOptions);
     if (browserPatchDisabled())
-        return origLaunch.call(self, withGVisorArgs(options));
+        return origLaunch.call(self, withSandboxArgs(options));
     const cdpUrl = rawLaunchCdpUrl(options, browserConfiguration);
     if (!cdpUrl) {
         if (!isAtHome())

package/dist/solver.js

@@ -31,7 +31,7 @@ exports.solve = solve;
 exports.reportSolveOutcome = reportSolveOutcome;
 exports.clearSolverStatusCache = clearSolverStatusCache;
 exports.DEGRADED_STATUS_MESSAGE = 'Site protection temporarily unavailable, results may be incomplete';
-const COMPAT_SUFFIX = '/compat/xhrdev';
+const COMPAT_SUFFIXES = ['/compat/v1', '/compat/xhrdev'];
 const STATUS_TTL_MS = 15_000;
 const MAX_BODY = 4 * 1024 * 1024;
 const UNAVAILABLE_CODES = new Set(['vendor_degraded', 'vendor_down', 'provider_unavailable', 'vendor_unsupported', 'solver_not_enabled']);
@@ -86,9 +86,12 @@ function solverEnv() {
 }
 function gateway() {
     const env = solverEnv();
-    if (!env || !env.url.endsWith(COMPAT_SUFFIX))
+    if (!env)
         return null;
-    return { base: env.url.slice(0, -COMPAT_SUFFIX.length), key: env.key };
+    const suffix = COMPAT_SUFFIXES.find((s) => env.url.endsWith(s));
+    if (!suffix)
+        return null;
+    return { base: env.url.slice(0, -suffix.length), key: env.key };
 }
 /** True when the platform solver API is available to this run. */
 function solverConfigured() {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.29",
+    "version": "1.2.32",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",