scrapely 1.2.30 → 1.2.33

package/dist/client-loader.d.ts

@@ -1,6 +1,6 @@
 /**
  * scrapely-client, loaded on first use (axios, proxy-agent and friends cost
- * ~0.7 s under gVisor, and most runs never need the API client).
+ * ~0.7 s on the platform, and most runs never need the API client).
  *
  * `USES_PUSH_DATA_INTERCEPTION` and `pushDataChargingContext` come from this
  * module (not declared in the SDK) so the SDK and the client share a single

package/dist/index.d.mts

@@ -11,6 +11,19 @@ export declare const BasicCrawler: typeof import("crawlee").BasicCrawler;
 export declare const BrowserCrawler: typeof import("crawlee").BrowserCrawler;
 export declare const JSDOMCrawler: typeof import("crawlee").JSDOMCrawler;
 export declare const LinkeDOMCrawler: typeof import("crawlee").LinkeDOMCrawler;
+export declare const AdaptivePlaywrightCrawler: typeof import("crawlee").AdaptivePlaywrightCrawler;
+export declare const RenderingTypePredictor: typeof import("crawlee").RenderingTypePredictor;
+export declare const SessionPool: typeof import("crawlee").SessionPool;
+export declare const Session: typeof import("crawlee").Session;
+export declare const AutoscaledPool: typeof import("crawlee").AutoscaledPool;
+export declare const createBasicRouter: typeof import("crawlee").createBasicRouter;
+export declare const createHttpRouter: typeof import("crawlee").createHttpRouter;
+export declare const createCheerioRouter: typeof import("crawlee").createCheerioRouter;
+export declare const createJSDOMRouter: typeof import("crawlee").createJSDOMRouter;
+export declare const createLinkeDOMRouter: typeof import("crawlee").createLinkeDOMRouter;
+export declare const createPlaywrightRouter: typeof import("crawlee").createPlaywrightRouter;
+export declare const createAdaptivePlaywrightRouter: typeof import("crawlee").createAdaptivePlaywrightRouter;
+export declare const createPuppeteerRouter: typeof import("crawlee").createPuppeteerRouter;
 export declare const ApifyClient: typeof import("scrapely-client").ScrapelyClient;
 export declare const playwrightUtils: typeof Sdk.playwrightUtils;
 export declare const puppeteerUtils: typeof Sdk.puppeteerUtils;

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 import './log-env.js';
 import type * as CrawleeBasic from '@crawlee/basic';
 import type * as CrawleeBrowser from '@crawlee/browser';
+import type * as CrawleeCore from '@crawlee/core';
 import type * as CrawleeCheerio from '@crawlee/cheerio';
 import type * as CrawleeHttp from '@crawlee/http';
 import type * as CrawleeJsdom from '@crawlee/jsdom';
@@ -27,6 +28,7 @@ export { ProxyConfiguration, isPlatformProxySentinel, validateProxySessionId } f
 export { createProxyConfiguration, getPlatformProxyEnv, type ProxyConfigurationOptions, type CreateProxyConfigurationOptions, type PlatformProxyEnv, type ProxyInfo, type ProxyConfigurationFunction, } from './proxy-configuration.js';
 export { BrowserConfiguration, createBrowserConfiguration, type BrowserConfigurationOptions, } from './browser-configuration.js';
 export { Dataset, KeyValueStore, RequestQueue, log, Log, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
+export { Request, RequestList, Router, RequestState, EventType, Statistics, Snapshotter, SystemStatus, NonRetryableError, RetryRequestError, MissingRouteError, CriticalError, useState, mergeCookies, } from '@crawlee/core';
 export type { LoggerOptions, KeyValueStoreIteratorOptions, KeyValueStoreOptions, DatasetConsumer, DatasetContent, DatasetDataOptions, DatasetIteratorOptions, DatasetMapper, DatasetOptions, DatasetReducer, QueueOperationInfo, RequestQueueOperationOptions, RequestQueueOptions, } from '@crawlee/core';
 export type { KeyConsumer } from './storage/key-value-store-client.js';
 export { utils, enqueueLinks, type EnqueueLinksOptions } from './utils.js';
@@ -47,5 +49,23 @@ export declare const JSDOMCrawler: typeof CrawleeJsdom.JSDOMCrawler;
 export type JSDOMCrawler = CrawleeJsdom.JSDOMCrawler;
 export declare const LinkeDOMCrawler: typeof CrawleeLinkedom.LinkeDOMCrawler;
 export type LinkeDOMCrawler = CrawleeLinkedom.LinkeDOMCrawler;
+export declare const SessionPool: typeof CrawleeCore.SessionPool;
+export type SessionPool = CrawleeCore.SessionPool;
+export declare const Session: typeof CrawleeCore.Session;
+export type Session = CrawleeCore.Session;
+export declare const AutoscaledPool: typeof CrawleeCore.AutoscaledPool;
+export type AutoscaledPool = CrawleeCore.AutoscaledPool;
+export declare const AdaptivePlaywrightCrawler: typeof CrawleePlaywright.AdaptivePlaywrightCrawler;
+export type AdaptivePlaywrightCrawler = CrawleePlaywright.AdaptivePlaywrightCrawler;
+export declare const RenderingTypePredictor: typeof CrawleePlaywright.RenderingTypePredictor;
+export type RenderingTypePredictor = CrawleePlaywright.RenderingTypePredictor;
+export declare const createBasicRouter: typeof CrawleeBasic.createBasicRouter;
+export declare const createHttpRouter: typeof CrawleeHttp.createHttpRouter;
+export declare const createCheerioRouter: typeof CrawleeCheerio.createCheerioRouter;
+export declare const createJSDOMRouter: typeof CrawleeJsdom.createJSDOMRouter;
+export declare const createLinkeDOMRouter: typeof CrawleeLinkedom.createLinkeDOMRouter;
+export declare const createPlaywrightRouter: typeof CrawleePlaywright.createPlaywrightRouter;
+export declare const createAdaptivePlaywrightRouter: typeof CrawleePlaywright.createAdaptivePlaywrightRouter;
+export declare const createPuppeteerRouter: typeof CrawleePuppeteer.createPuppeteerRouter;
 export declare const playwrightUtils: typeof CrawleePlaywright.playwrightUtils;
 export declare const puppeteerUtils: typeof CrawleePuppeteer.puppeteerUtils;

package/dist/index.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Logger = exports.LogLevel = exports.Log = exports.log = exports.RequestQueue = exports.KeyValueStore = exports.Dataset = exports.createBrowserConfiguration = exports.BrowserConfiguration = exports.getPlatformProxyEnv = exports.createProxyConfiguration = exports.validateProxySessionId = exports.isPlatformProxySentinel = exports.ProxyConfiguration = exports.ApiStorageClientFactory = exports.BillingTracker = exports.createChargingClient = exports.validateRowCharges = exports.pushDataWithCharges = exports.pushDataAndCharge = exports.mergeChargeResults = exports.resolveDefaultDatasetItemEventName = exports.isDefaultDatasetItemEvent = exports.hasDefaultDatasetItemEventInPricing = exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = exports.APIFY_DEFAULT_DATASET_ITEM_EVENT = exports.DEFAULT_DATASET_ITEM_EVENT = exports.ChargeRejectedError = exports.ChargeUncertainError = exports.ChargingManager = exports.PlatformEventManager = exports.stepAllowed = exports.solverStatus = exports.solverConfigured = exports.solve = exports.reportSolveOutcome = exports.isBlocked = exports.detectVendor = exports.degradedStatusMessage = exports.VendorUnavailable = exports.SolverRateLimited = exports.SolverAuthError = exports.SolveAttemptError = exports.IpBanned = exports.DEGRADED_STATUS_MESSAGE = exports.Configuration = exports.EXIT_CODES = exports.ACTOR_PERMISSION_LEVEL = exports.Actor = exports.setCdpUrl = void 0;
-exports.ScrapelyRequestList = exports.enqueueLinks = exports.utils = exports.LoggerText = exports.LoggerJson = void 0;
+exports.ScrapelyRequestList = exports.enqueueLinks = exports.utils = exports.mergeCookies = exports.useState = exports.CriticalError = exports.MissingRouteError = exports.RetryRequestError = exports.NonRetryableError = exports.SystemStatus = exports.Snapshotter = exports.Statistics = exports.RequestState = exports.Router = exports.RequestList = exports.Request = exports.LoggerText = exports.LoggerJson = void 0;
 require("./log-env.js");
 const lazy_exports_js_1 = require("./lazy-exports.js");
 var patch_browser_pool_js_1 = require("./patch-browser-pool.js");
@@ -74,6 +74,22 @@ Object.defineProperty(exports, "LogLevel", { enumerable: true, get: function ()
 Object.defineProperty(exports, "Logger", { enumerable: true, get: function () { return core_1.Logger; } });
 Object.defineProperty(exports, "LoggerJson", { enumerable: true, get: function () { return core_1.LoggerJson; } });
 Object.defineProperty(exports, "LoggerText", { enumerable: true, get: function () { return core_1.LoggerText; } });
+// Cheap core classes the reference SDK re-exports from crawlee. @crawlee/core is
+// already loaded by the exports above, so these are plain re-exports.
+var core_2 = require("@crawlee/core");
+Object.defineProperty(exports, "Request", { enumerable: true, get: function () { return core_2.Request; } });
+Object.defineProperty(exports, "RequestList", { enumerable: true, get: function () { return core_2.RequestList; } });
+Object.defineProperty(exports, "Router", { enumerable: true, get: function () { return core_2.Router; } });
+Object.defineProperty(exports, "RequestState", { enumerable: true, get: function () { return core_2.RequestState; } });
+Object.defineProperty(exports, "Statistics", { enumerable: true, get: function () { return core_2.Statistics; } });
+Object.defineProperty(exports, "Snapshotter", { enumerable: true, get: function () { return core_2.Snapshotter; } });
+Object.defineProperty(exports, "SystemStatus", { enumerable: true, get: function () { return core_2.SystemStatus; } });
+Object.defineProperty(exports, "NonRetryableError", { enumerable: true, get: function () { return core_2.NonRetryableError; } });
+Object.defineProperty(exports, "RetryRequestError", { enumerable: true, get: function () { return core_2.RetryRequestError; } });
+Object.defineProperty(exports, "MissingRouteError", { enumerable: true, get: function () { return core_2.MissingRouteError; } });
+Object.defineProperty(exports, "CriticalError", { enumerable: true, get: function () { return core_2.CriticalError; } });
+Object.defineProperty(exports, "useState", { enumerable: true, get: function () { return core_2.useState; } });
+Object.defineProperty(exports, "mergeCookies", { enumerable: true, get: function () { return core_2.mergeCookies; } });
 // Apify v2 compatible utils namespace
 var utils_js_1 = require("./utils.js");
 Object.defineProperty(exports, "utils", { enumerable: true, get: function () { return utils_js_1.utils; } });

package/dist/index.mjs

@@ -24,6 +24,19 @@ export const BasicCrawler = lazyExport('BasicCrawler', 'class');
 export const BrowserCrawler = lazyExport('BrowserCrawler', 'class');
 export const JSDOMCrawler = lazyExport('JSDOMCrawler', 'class');
 export const LinkeDOMCrawler = lazyExport('LinkeDOMCrawler', 'class');
+export const AdaptivePlaywrightCrawler = lazyExport('AdaptivePlaywrightCrawler', 'class');
+export const RenderingTypePredictor = lazyExport('RenderingTypePredictor', 'class');
+export const SessionPool = lazyExport('SessionPool', 'class');
+export const Session = lazyExport('Session', 'class');
+export const AutoscaledPool = lazyExport('AutoscaledPool', 'class');
+export const createBasicRouter = lazyExport('createBasicRouter', 'class');
+export const createHttpRouter = lazyExport('createHttpRouter', 'class');
+export const createCheerioRouter = lazyExport('createCheerioRouter', 'class');
+export const createJSDOMRouter = lazyExport('createJSDOMRouter', 'class');
+export const createLinkeDOMRouter = lazyExport('createLinkeDOMRouter', 'class');
+export const createPlaywrightRouter = lazyExport('createPlaywrightRouter', 'class');
+export const createAdaptivePlaywrightRouter = lazyExport('createAdaptivePlaywrightRouter', 'class');
+export const createPuppeteerRouter = lazyExport('createPuppeteerRouter', 'class');
 export const ApifyClient = lazyExport('ApifyClient', 'class');
 export const playwrightUtils = lazyExport('playwrightUtils', 'object');
 export const puppeteerUtils = lazyExport('puppeteerUtils', 'object');

package/dist/lazy-exports.js

@@ -7,7 +7,7 @@ exports.lazyExport = lazyExport;
 /**
  * Public SDK names that load heavy packages: Crawlee's crawlers (jsdom,
  * linkedom, cheerio, the browser pool, playwright) and scrapely-client.
- * Importing the SDK used to load all of them, ~4 s at 1 core under gVisor;
+ * Importing the SDK used to load all of them, ~4 s at 1 core on the platform;
  * they now load on first access.
  *
  * - CommonJS (`require('scrapely').PlaywrightCrawler`): a getter on the
@@ -30,6 +30,19 @@ exports.LAZY_EXPORTS = {
     BrowserCrawler: ['@crawlee/browser', 'BrowserCrawler'],
     JSDOMCrawler: ['@crawlee/jsdom', 'JSDOMCrawler'],
     LinkeDOMCrawler: ['@crawlee/linkedom', 'LinkeDOMCrawler'],
+    AdaptivePlaywrightCrawler: ['@crawlee/playwright', 'AdaptivePlaywrightCrawler'],
+    RenderingTypePredictor: ['@crawlee/playwright', 'RenderingTypePredictor'],
+    SessionPool: ['@crawlee/core', 'SessionPool'],
+    Session: ['@crawlee/core', 'Session'],
+    AutoscaledPool: ['@crawlee/core', 'AutoscaledPool'],
+    createBasicRouter: ['@crawlee/basic', 'createBasicRouter'],
+    createHttpRouter: ['@crawlee/http', 'createHttpRouter'],
+    createCheerioRouter: ['@crawlee/cheerio', 'createCheerioRouter'],
+    createJSDOMRouter: ['@crawlee/jsdom', 'createJSDOMRouter'],
+    createLinkeDOMRouter: ['@crawlee/linkedom', 'createLinkeDOMRouter'],
+    createPlaywrightRouter: ['@crawlee/playwright', 'createPlaywrightRouter'],
+    createAdaptivePlaywrightRouter: ['@crawlee/playwright', 'createAdaptivePlaywrightRouter'],
+    createPuppeteerRouter: ['@crawlee/puppeteer', 'createPuppeteerRouter'],
     playwrightUtils: ['@crawlee/playwright', 'playwrightUtils'],
     puppeteerUtils: ['@crawlee/puppeteer', 'puppeteerUtils'],
     ApifyClient: ['scrapely-client', 'ApifyClient'],

package/dist/node-wrapper.d.ts

@@ -4,7 +4,7 @@
  * The base images preload the SDK's browser patch into every Node process
  * (NODE_OPTIONS=--import /opt/scrapely/register.mjs). `npm start` is two Node
  * processes: npm, which only spawns the start script, and the actor. npm itself
- * costs ~0.6 s at 1 core and ~2.7 s at 0.25 core (1 GB run) under gVisor.
+ * costs ~0.6 s at 1 core and ~2.7 s at 0.25 core (1 GB run) on the platform.
  *
  * - npm / npx / yarn / pnpm processes never run actor code, so they get no
  *   loader hooks. Their children inherit NODE_OPTIONS and get the preload.

package/dist/patch-browser-pool.d.ts

@@ -66,7 +66,7 @@ export declare function patchLoadedCrawleeCopies(): void;
  * - `cdpUrl` becomes a valid launcher option (PlaywrightLauncher and
  *   PuppeteerLauncher copy BrowserLauncher's shape when their file loads, so
  *   each class gets it; browser-launcher.js loads first, so the copies inherit it).
- * - Local launches get gVisor-safe Chrome args under runsc. Playwright/Puppeteer
+ * - Local launches get sandbox-safe Chrome args on the platform. Playwright/Puppeteer
  *   pool launches use CDP and skip this.
  *
  * @internal Exported for tests.

package/dist/patch-browser-pool.js

@@ -60,20 +60,20 @@ function isAtHome() {
 function browserPatchDisabled() {
     return process.env.SCRAPELY_BROWSER_PATCH === '0';
 }
-/** gVisor lacks crashpad support; ReadonlyRootfs + tmpfs /tmp loses crash dirs. */
-const GVISOR_CHROME_ARGS = [
+/** Platform sandbox: no crash reporter; read-only rootfs + tmpfs /tmp loses crash dirs. */
+const SANDBOX_CHROME_ARGS = [
     '--disable-breakpad',
     '--disable-crash-reporter',
     '--disable-dev-shm-usage',
     '--crash-dumps-dir=/tmp',
 ];
-/** Add gVisor-safe Chrome args to local launch options when running under runsc. */
-function withGVisorArgs(options) {
+/** Add sandbox-safe Chrome args to local launch options when running on the platform. */
+function withSandboxArgs(options) {
     const opts = { ...(options ?? {}) };
-    if (process.env.SCRAPELY_CONTAINER_RUNTIME !== 'runsc')
+    if (!isAtHome())
         return opts;
     const args = [...(opts.args ?? [])];
-    for (const arg of GVISOR_CHROME_ARGS) {
+    for (const arg of SANDBOX_CHROME_ARGS) {
         if (!args.includes(arg))
             args.push(arg);
     }
@@ -84,7 +84,7 @@ function withGVisorArgs(options) {
  * Require CDP_URL for platform browser runs. Local launch is disabled on the
  * platform unless the build is unpatched (SCRAPELY_BROWSER_PATCH=0) — otherwise
  * a worker container without CDP_URL is a misconfiguration and must fail
- * loudly instead of launching Chromium inside the (gVisor) container.
+ * loudly instead of launching Chromium inside the platform container.
  */
 function requireCdpUrl() {
     const cdpUrl = getCdpUrl();
@@ -491,11 +491,11 @@ function patchLoadedCrawleeCopies() {
         }
     }
 }
-// ─── Crawlee launchers: `cdpUrl` option + gVisor-safe local launch args ─────
+// ─── Crawlee launchers: `cdpUrl` option + sandbox-safe local launch args ─────
 //
 // Applied when a launcher module loads (Module._load hook below), never by
 // importing Crawlee up front: loading @crawlee/browser-pool and playwright
-// eagerly cost ~2.4 s per Node process under gVisor.
+// eagerly cost ~2.4 s per Node process on the platform.
 /** Requests that load a Crawlee launcher file (`require('./internals/browser-launcher')`). */
 const LAUNCHER_REQUEST_RE = /(^|[/\\])internals[/\\](browser|playwright|puppeteer)-launcher(\.js)?$/;
 /** Cached module filenames of Crawlee launcher files (any copy). */
@@ -533,7 +533,7 @@ function addCdpUrlOption(Launcher, parentFilename) {
  * - `cdpUrl` becomes a valid launcher option (PlaywrightLauncher and
  *   PuppeteerLauncher copy BrowserLauncher's shape when their file loads, so
  *   each class gets it; browser-launcher.js loads first, so the copies inherit it).
- * - Local launches get gVisor-safe Chrome args under runsc. Playwright/Puppeteer
+ * - Local launches get sandbox-safe Chrome args on the platform. Playwright/Puppeteer
  *   pool launches use CDP and skip this.
  *
  * @internal Exported for tests.
@@ -555,17 +555,17 @@ function patchBrowserLauncherModule(mod, parentFilename) {
     }
     const BrowserLauncher = mod.BrowserLauncher;
     const proto = BrowserLauncher?.prototype;
-    if (!proto || typeof proto.createLaunchOptions !== 'function' || proto._scrapelyGVisorArgsPatched)
+    if (!proto || typeof proto.createLaunchOptions !== 'function' || proto._scrapelySandboxArgsPatched)
         return;
     const origCreateLaunchOptions = proto.createLaunchOptions;
     proto.createLaunchOptions = function (...args) {
         const launchOptions = origCreateLaunchOptions.apply(this, args);
-        // gVisor doesn't support Chrome's crashpad_handler process. Also needed
+        // The sandbox has no Chrome crashpad_handler support. Also needed
         // with ReadonlyRootfs + tmpfs /tmp where /tmp/.X11-unix and the crashpad
         // database dir don't persist from the base image.
-        if (process.env.SCRAPELY_CONTAINER_RUNTIME === 'runsc' && launchOptions) {
+        if (isAtHome() && launchOptions) {
             launchOptions.args = launchOptions.args || [];
-            for (const arg of GVISOR_CHROME_ARGS) {
+            for (const arg of SANDBOX_CHROME_ARGS) {
                 if (!launchOptions.args.includes(arg)) {
                     launchOptions.args.push(arg);
                 }
@@ -573,7 +573,7 @@ function patchBrowserLauncherModule(mod, parentFilename) {
         }
         return launchOptions;
     };
-    proto._scrapelyGVisorArgsPatched = true;
+    proto._scrapelySandboxArgsPatched = true;
 }
 /**
  * Split our `browserConfiguration` option off raw launch options. The options
@@ -655,7 +655,7 @@ function patchRawBrowserType(chromium, browserType, name) {
     browserType.launch = async function rawLaunchPatched(launchOptions = {}) {
         const { options, browserConfiguration } = takeBrowserConfiguration(launchOptions);
         if (browserPatchDisabled())
-            return origLaunch.call(this, withGVisorArgs(options));
+            return origLaunch.call(this, withSandboxArgs(options));
         const cdpUrl = rawLaunchCdpUrl(options, browserConfiguration);
         if (!cdpUrl) {
             if (!isAtHome())
@@ -674,7 +674,7 @@ function patchRawBrowserType(chromium, browserType, name) {
             const { options, browserConfiguration } = takeBrowserConfiguration(lpcArgs[optionsIndex]);
             const withOptions = (opts) => optionsIndex === 1 ? [userDataDir, opts] : [opts];
             if (browserPatchDisabled())
-                return origLpc.call(this, ...withOptions(withGVisorArgs(options)));
+                return origLpc.call(this, ...withOptions(withSandboxArgs(options)));
             const cdpUrl = rawLaunchCdpUrl(options, browserConfiguration);
             if (!cdpUrl) {
                 if (!isAtHome()) {
@@ -719,7 +719,7 @@ function patchRawPlaywrightModule(pw) {
 async function puppeteerLaunchPatched(self, origLaunch, moduleName, launchOptions = {}) {
     const { options, browserConfiguration } = takeBrowserConfiguration(launchOptions);
     if (browserPatchDisabled())
-        return origLaunch.call(self, withGVisorArgs(options));
+        return origLaunch.call(self, withSandboxArgs(options));
     const cdpUrl = rawLaunchCdpUrl(options, browserConfiguration);
     if (!cdpUrl) {
         if (!isAtHome())

package/dist/solver.js

@@ -31,7 +31,7 @@ exports.solve = solve;
 exports.reportSolveOutcome = reportSolveOutcome;
 exports.clearSolverStatusCache = clearSolverStatusCache;
 exports.DEGRADED_STATUS_MESSAGE = 'Site protection temporarily unavailable, results may be incomplete';
-const COMPAT_SUFFIXES = ['/compat/v1', '/compat/xhrdev'];
+const COMPAT_SUFFIXES = ['/compat/v1'];
 const STATUS_TTL_MS = 15_000;
 const MAX_BODY = 4 * 1024 * 1024;
 const UNAVAILABLE_CODES = new Set(['vendor_degraded', 'vendor_down', 'provider_unavailable', 'vendor_unsupported', 'solver_not_enabled']);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.30",
+    "version": "1.2.33",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",

package/dist/runner/docker-manager.d.ts

@@ -1,88 +0,0 @@
-import Docker from 'dockerode';
-import { TimeoutManager } from './timeout-manager.js';
-import { DragonflyClient } from '../storage/dragonfly-client.js';
-import type { UserPlan } from '../storage/ttl-manager.js';
-export interface RunOptions {
-    /** Path to the actor package directory */
-    packagePath: string;
-    /** Path to the .env file */
-    envPath?: string;
-    /** Environment variables to pass to container */
-    env?: Record<string, string>;
-    /** Maximum memory in MB (default: from manifest or 512) */
-    memoryMb?: number;
-    /** Maximum timeout in seconds (default: from manifest or 1800) */
-    timeout?: number;
-    /** Run ID for tracking */
-    runId?: string;
-    /** User ID */
-    userId?: string;
-    /** User plan for TTL */
-    userPlan?: UserPlan;
-    /** Dragonfly client for storage cleanup */
-    dragonflyClient?: DragonflyClient;
-}
-export interface RunResult {
-    containerId: string;
-    exitCode: number;
-    logs: string;
-    status: 'success' | 'failed' | 'timed_out' | 'killed';
-    duration: number;
-    /** Storage IDs that were cleaned up */
-    cleanedUp?: {
-        requestQueueDeleted: boolean;
-        logsUploadedToS3: boolean;
-    };
-}
-export declare class DockerManager {
-    private docker;
-    private timeoutManager;
-    private imageName;
-    constructor(options?: {
-        socketPath?: string;
-        imageName?: string;
-    });
-    /**
-     * Run an actor package in a container
-     */
-    run(options: RunOptions): Promise<RunResult>;
-    /**
-     * Cleanup storage after run completes.
-     * - Deletes request queue immediately (no longer needed)
-     * - Uploads logs to S3 for long-term storage
-     * - Updates run metadata with final status
-     */
-    private cleanupAfterRun;
-    /**
-     * Stop a container (SIGTERM)
-     */
-    stop(containerId: string, timeoutSeconds?: number): Promise<void>;
-    /**
-     * Kill a container (SIGKILL)
-     */
-    kill(containerId: string): Promise<void>;
-    /**
-     * Get logs from a container
-     */
-    getLogs(containerId: string): Promise<string>;
-    /**
-     * Get container status
-     */
-    getStatus(containerId: string): Promise<string | null>;
-    /**
-     * List all scrapely containers
-     */
-    listContainers(): Promise<Docker.ContainerInfo[]>;
-    /**
-     * Get the timeout manager
-     */
-    getTimeoutManager(): TimeoutManager;
-    /**
-     * Track internal data transfer from container network stats.
-     * Following Apify's model:
-     * - Ingress (rx_bytes) is FREE - scraped content coming in
-     * - Internal transfer is tracked separately in storage clients
-     * - External egress is tracked in API layer when users fetch results
-     */
-    private trackDataTransfer;
-}

package/dist/runner/docker-manager.js

@@ -1,283 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DockerManager = void 0;
-const dockerode_1 = __importDefault(require("dockerode"));
-const timeout_manager_js_1 = require("./timeout-manager.js");
-const validator_js_1 = require("../validator.js");
-const path_1 = __importDefault(require("path"));
-const run_metadata_js_1 = require("../storage/run-metadata.js");
-const log_manager_js_1 = require("../storage/log-manager.js");
-class DockerManager {
-    docker;
-    timeoutManager;
-    imageName;
-    constructor(options) {
-        this.docker = new dockerode_1.default({
-            socketPath: options?.socketPath || '/var/run/docker.sock',
-        });
-        this.timeoutManager = new timeout_manager_js_1.TimeoutManager();
-        this.imageName = options?.imageName || 'scrapely-runner:latest';
-        // Set up timeout event handlers
-        this.timeoutManager.on('timeout', async ({ containerId }) => {
-            console.log(`[DockerManager] Container ${containerId} timed out, sending SIGTERM...`);
-            await this.stop(containerId, 30);
-        });
-        this.timeoutManager.on('kill', async ({ containerId }) => {
-            console.log(`[DockerManager] Container ${containerId} grace period expired, sending SIGKILL...`);
-            await this.kill(containerId);
-        });
-    }
-    /**
-     * Run an actor package in a container
-     */
-    async run(options) {
-        // Validate package
-        const validation = (0, validator_js_1.validatePackage)(options.packagePath);
-        if (!validation.valid) {
-            throw new Error(`Invalid package: ${validation.errors.join(', ')}`);
-        }
-        const manifest = validation.manifest;
-        const startTime = Date.now();
-        const memoryMb = options.memoryMb || manifest.maxMemory || 512;
-        const timeout = options.timeout || manifest.maxTimeout || 1800;
-        // Build environment variables
-        const envVars = [];
-        if (options.env) {
-            for (const [key, value] of Object.entries(options.env)) {
-                envVars.push(`${key}=${value}`);
-            }
-        }
-        // Build volume mounts
-        const binds = [
-            `${path_1.default.resolve(options.packagePath)}:/actor/package:ro`,
-        ];
-        if (options.envPath) {
-            binds.push(`${path_1.default.resolve(options.envPath)}:/actor/.env:ro`);
-        }
-        // Create container
-        console.log(`[DockerManager] Creating container for ${manifest.name}@${manifest.version}...`);
-        const container = await this.docker.createContainer({
-            Image: this.imageName,
-            Env: envVars,
-            HostConfig: {
-                Binds: binds,
-                Memory: memoryMb * 1024 * 1024, // Convert MB to bytes
-                AutoRemove: false, // We'll remove it manually after getting logs
-            },
-            Labels: {
-                'scrapely.actor.name': manifest.name,
-                'scrapely.actor.version': manifest.version,
-            },
-        });
-        const containerId = container.id;
-        console.log(`[DockerManager] Container created: ${containerId.substring(0, 12)}`);
-        // Start timeout timer
-        const timeoutOptions = {
-            maxTimeout: timeout,
-            gracePeriod: 30,
-        };
-        this.timeoutManager.startTimer(containerId, timeoutOptions);
-        // Set log TTL now so the key has the correct expiry from the first byte written
-        if (options.dragonflyClient && options.runId) {
-            const logManager = new log_manager_js_1.LogManager(options.dragonflyClient);
-            await logManager.setLogTtl(options.runId, options.userPlan ?? 'free');
-        }
-        // Start container
-        await container.start();
-        console.log(`[DockerManager] Container started with ${timeout}s timeout`);
-        // Wait for container to finish
-        let exitCode = 0;
-        let status = 'success';
-        try {
-            const result = await container.wait();
-            exitCode = result.StatusCode || 0;
-            // Check if it was timed out
-            const timerState = this.timeoutManager.getState(containerId);
-            if (timerState?.status === 'killed') {
-                status = 'killed';
-            }
-            else if (timerState?.status === 'timed_out') {
-                status = 'timed_out';
-            }
-            else if (exitCode !== 0) {
-                status = 'failed';
-            }
-            // Stop the timer
-            this.timeoutManager.stopTimer(containerId);
-        }
-        catch (error) {
-            // Container might have been killed
-            const timerState = this.timeoutManager.getState(containerId);
-            if (timerState?.status === 'killed') {
-                status = 'killed';
-                exitCode = 137; // SIGKILL exit code
-            }
-            else {
-                status = 'failed';
-                exitCode = 1;
-            }
-            this.timeoutManager.stopTimer(containerId);
-        }
-        // Get logs
-        const logs = await this.getLogs(containerId);
-        // Get network stats and track data transfer before removing container
-        await this.trackDataTransfer(container, options);
-        // Remove container
-        try {
-            await container.remove();
-        }
-        catch (e) {
-            // Ignore removal errors
-        }
-        const duration = Math.round((Date.now() - startTime) / 1000);
-        console.log(`[DockerManager] Container finished: ${status} (exit code: ${exitCode}, duration: ${duration}s)`);
-        // Cleanup after run completes
-        const cleanedUp = await this.cleanupAfterRun(options, status);
-        return {
-            containerId,
-            exitCode,
-            logs,
-            status,
-            duration,
-            cleanedUp,
-        };
-    }
-    /**
-     * Cleanup storage after run completes.
-     * - Deletes request queue immediately (no longer needed)
-     * - Uploads logs to S3 for long-term storage
-     * - Updates run metadata with final status
-     */
-    async cleanupAfterRun(options, status) {
-        const result = { requestQueueDeleted: false, logsUploadedToS3: false };
-        if (!options.dragonflyClient || !options.runId) {
-            return result;
-        }
-        try {
-            // Get run metadata to find storage IDs
-            const metadataManager = new run_metadata_js_1.RunMetadataManager(options.dragonflyClient);
-            const metadata = await metadataManager.get(options.runId);
-            if (metadata) {
-                // Delete request queue immediately - it's only useful during execution
-                const rqClient = options.dragonflyClient.requestQueue(metadata.storageIds.requestQueue);
-                await rqClient.delete();
-                result.requestQueueDeleted = true;
-                console.log(`[DockerManager] Deleted request queue: ${metadata.storageIds.requestQueue}`);
-                // Upload logs to S3 for long-term storage
-                const logManager = new log_manager_js_1.LogManager(options.dragonflyClient);
-                const s3Key = await logManager.uploadToS3(options.runId, metadata.userId);
-                result.logsUploadedToS3 = s3Key !== null;
-                // Delete logs from Redis now that they're in S3 — saves RAM.
-                // If S3 upload wasn't configured/failed, TTL set at run start handles expiry.
-                if (result.logsUploadedToS3) {
-                    await logManager.deleteLogs(options.runId);
-                    console.log(`[DockerManager] Deleted Redis logs for run ${options.runId} (persisted to S3)`);
-                }
-                // Update run metadata with final status
-                const runStatus = status === 'success' ? 'success'
-                    : status === 'timed_out' ? 'timed_out'
-                        : status === 'killed' ? 'killed'
-                            : 'failed';
-                await metadataManager.finish(options.runId, runStatus);
-            }
-        }
-        catch (error) {
-            console.error(`[DockerManager] Cleanup error: ${error.message}`);
-        }
-        return result;
-    }
-    /**
-     * Stop a container (SIGTERM)
-     */
-    async stop(containerId, timeoutSeconds = 30) {
-        const container = this.docker.getContainer(containerId);
-        try {
-            await container.stop({ t: timeoutSeconds });
-        }
-        catch (error) {
-            if (error.statusCode !== 304) { // Not modified - already stopped
-                throw error;
-            }
-        }
-    }
-    /**
-     * Kill a container (SIGKILL)
-     */
-    async kill(containerId) {
-        const container = this.docker.getContainer(containerId);
-        try {
-            await container.kill();
-        }
-        catch (error) {
-            if (error.statusCode !== 409) { // Not running
-                throw error;
-            }
-        }
-    }
-    /**
-     * Get logs from a container
-     */
-    async getLogs(containerId) {
-        const container = this.docker.getContainer(containerId);
-        try {
-            const logStream = await container.logs({
-                stdout: true,
-                stderr: true,
-                timestamps: false,
-            });
-            // Docker logs are prefixed with headers, clean them
-            return logStream.toString('utf-8').replace(/\x00\x00\x00[\x00-\xff][\x00-\xff][\x00-\xff]/g, '');
-        }
-        catch (error) {
-            return '';
-        }
-    }
-    /**
-     * Get container status
-     */
-    async getStatus(containerId) {
-        const container = this.docker.getContainer(containerId);
-        try {
-            const info = await container.inspect();
-            return info.State.Status;
-        }
-        catch (error) {
-            return null;
-        }
-    }
-    /**
-     * List all scrapely containers
-     */
-    async listContainers() {
-        const containers = await this.docker.listContainers({
-            all: true,
-            filters: {
-                label: ['scrapely.actor.name'],
-            },
-        });
-        return containers;
-    }
-    /**
-     * Get the timeout manager
-     */
-    getTimeoutManager() {
-        return this.timeoutManager;
-    }
-    /**
-     * Track internal data transfer from container network stats.
-     * Following Apify's model:
-     * - Ingress (rx_bytes) is FREE - scraped content coming in
-     * - Internal transfer is tracked separately in storage clients
-     * - External egress is tracked in API layer when users fetch results
-     */
-    async trackDataTransfer(container, options) {
-        // Ingress is free - no billing for data coming into the platform from internet
-        // Internal transfer (actor ↔ storage) is tracked in storage clients
-        // External egress (platform → user) is tracked in API routes
-        // This method is kept for future use if needed
-    }
-}
-exports.DockerManager = DockerManager;

package/dist/runner/index.d.ts

@@ -1,2 +0,0 @@
-export { DockerManager, RunOptions, RunResult } from './docker-manager.js';
-export { TimeoutManager, TimeoutOptions, TimerState } from './timeout-manager.js';

package/dist/runner/index.js

@@ -1,7 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TimeoutManager = exports.DockerManager = void 0;
-var docker_manager_js_1 = require("./docker-manager.js");
-Object.defineProperty(exports, "DockerManager", { enumerable: true, get: function () { return docker_manager_js_1.DockerManager; } });
-var timeout_manager_js_1 = require("./timeout-manager.js");
-Object.defineProperty(exports, "TimeoutManager", { enumerable: true, get: function () { return timeout_manager_js_1.TimeoutManager; } });

package/dist/runner/timeout-manager.d.ts

@@ -1,43 +0,0 @@
-import { EventEmitter } from 'events';
-export interface TimeoutOptions {
-    /** Maximum runtime in seconds */
-    maxTimeout: number;
-    /** Grace period after SIGTERM before SIGKILL (default: 30s) */
-    gracePeriod?: number;
-}
-export interface TimerState {
-    containerId: string;
-    startedAt: Date;
-    timeoutAt: Date;
-    status: 'running' | 'stopping' | 'timed_out' | 'completed' | 'killed';
-    timeoutTimer?: NodeJS.Timeout;
-    killTimer?: NodeJS.Timeout;
-}
-export declare class TimeoutManager extends EventEmitter {
-    private timers;
-    private defaultGracePeriod;
-    /**
-     * Start a timeout timer for a container
-     */
-    startTimer(containerId: string, options: TimeoutOptions): TimerState;
-    /**
-     * Stop a timer (container completed successfully)
-     */
-    stopTimer(containerId: string): void;
-    /**
-     * Get timer state for a container
-     */
-    getState(containerId: string): TimerState | undefined;
-    /**
-     * Get remaining time in seconds for a container
-     */
-    getRemainingTime(containerId: string): number | undefined;
-    /**
-     * Get all active timers
-     */
-    getActiveTimers(): TimerState[];
-    /**
-     * Clear all timers
-     */
-    clearAll(): void;
-}

package/dist/runner/timeout-manager.js

@@ -1,90 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TimeoutManager = void 0;
-const events_1 = require("events");
-class TimeoutManager extends events_1.EventEmitter {
-    timers = new Map();
-    defaultGracePeriod = 30; // seconds
-    /**
-     * Start a timeout timer for a container
-     */
-    startTimer(containerId, options) {
-        // Stop existing timer if any
-        this.stopTimer(containerId);
-        const startedAt = new Date();
-        const timeoutAt = new Date(startedAt.getTime() + options.maxTimeout * 1000);
-        const gracePeriod = options.gracePeriod ?? this.defaultGracePeriod;
-        const state = {
-            containerId,
-            startedAt,
-            timeoutAt,
-            status: 'running',
-        };
-        // Set up timeout timer
-        state.timeoutTimer = setTimeout(async () => {
-            this.emit('timeout', { containerId, state });
-            state.status = 'timed_out';
-            this.emit('stopping', { containerId, gracePeriod });
-            // Set up kill timer after timeout
-            state.killTimer = setTimeout(() => {
-                state.status = 'killed';
-                this.emit('kill', { containerId });
-                this.timers.delete(containerId);
-            }, gracePeriod * 1000);
-        }, options.maxTimeout * 1000);
-        this.timers.set(containerId, state);
-        this.emit('started', { containerId, timeoutAt });
-        return state;
-    }
-    /**
-     * Stop a timer (container completed successfully)
-     */
-    stopTimer(containerId) {
-        const state = this.timers.get(containerId);
-        if (!state)
-            return;
-        // Clear timers
-        if (state.timeoutTimer) {
-            clearTimeout(state.timeoutTimer);
-        }
-        if (state.killTimer) {
-            clearTimeout(state.killTimer);
-        }
-        if (state.status === 'running') {
-            state.status = 'completed';
-            this.emit('completed', { containerId, state });
-        }
-        this.timers.delete(containerId);
-    }
-    /**
-     * Get timer state for a container
-     */
-    getState(containerId) {
-        return this.timers.get(containerId);
-    }
-    /**
-     * Get remaining time in seconds for a container
-     */
-    getRemainingTime(containerId) {
-        const state = this.timers.get(containerId);
-        if (!state || state.status !== 'running')
-            return undefined;
-        const remaining = Math.max(0, (state.timeoutAt.getTime() - Date.now()) / 1000);
-        return remaining;
-    }
-    /**
-     * Get all active timers
-     */
-    getActiveTimers() {
-        return Array.from(this.timers.values()).filter(s => s.status === 'running');
-    }
-    /**
-     * Clear all timers
-     */
-    clearAll() {
-        for (const [containerId] of this.timers) {
-            this.stopTimer(containerId);
-        }
-    }
-}
-exports.TimeoutManager = TimeoutManager;