scrapely 1.2.3 → 1.2.4

package/dist/runner/docker-manager.js

@@ -0,0 +1,283 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DockerManager = void 0;
+const dockerode_1 = __importDefault(require("dockerode"));
+const timeout_manager_js_1 = require("./timeout-manager.js");
+const validator_js_1 = require("../validator.js");
+const path_1 = __importDefault(require("path"));
+const run_metadata_js_1 = require("../storage/run-metadata.js");
+const log_manager_js_1 = require("../storage/log-manager.js");
+class DockerManager {
+    docker;
+    timeoutManager;
+    imageName;
+    constructor(options) {
+        this.docker = new dockerode_1.default({
+            socketPath: options?.socketPath || '/var/run/docker.sock',
+        });
+        this.timeoutManager = new timeout_manager_js_1.TimeoutManager();
+        this.imageName = options?.imageName || 'scrapely-runner:latest';
+        // Set up timeout event handlers
+        this.timeoutManager.on('timeout', async ({ containerId }) => {
+            console.log(`[DockerManager] Container ${containerId} timed out, sending SIGTERM...`);
+            await this.stop(containerId, 30);
+        });
+        this.timeoutManager.on('kill', async ({ containerId }) => {
+            console.log(`[DockerManager] Container ${containerId} grace period expired, sending SIGKILL...`);
+            await this.kill(containerId);
+        });
+    }
+    /**
+     * Run an actor package in a container
+     */
+    async run(options) {
+        // Validate package
+        const validation = (0, validator_js_1.validatePackage)(options.packagePath);
+        if (!validation.valid) {
+            throw new Error(`Invalid package: ${validation.errors.join(', ')}`);
+        }
+        const manifest = validation.manifest;
+        const startTime = Date.now();
+        const memoryMb = options.memoryMb || manifest.maxMemory || 512;
+        const timeout = options.timeout || manifest.maxTimeout || 1800;
+        // Build environment variables
+        const envVars = [];
+        if (options.env) {
+            for (const [key, value] of Object.entries(options.env)) {
+                envVars.push(`${key}=${value}`);
+            }
+        }
+        // Build volume mounts
+        const binds = [
+            `${path_1.default.resolve(options.packagePath)}:/actor/package:ro`,
+        ];
+        if (options.envPath) {
+            binds.push(`${path_1.default.resolve(options.envPath)}:/actor/.env:ro`);
+        }
+        // Create container
+        console.log(`[DockerManager] Creating container for ${manifest.name}@${manifest.version}...`);
+        const container = await this.docker.createContainer({
+            Image: this.imageName,
+            Env: envVars,
+            HostConfig: {
+                Binds: binds,
+                Memory: memoryMb * 1024 * 1024, // Convert MB to bytes
+                AutoRemove: false, // We'll remove it manually after getting logs
+            },
+            Labels: {
+                'scrapely.actor.name': manifest.name,
+                'scrapely.actor.version': manifest.version,
+            },
+        });
+        const containerId = container.id;
+        console.log(`[DockerManager] Container created: ${containerId.substring(0, 12)}`);
+        // Start timeout timer
+        const timeoutOptions = {
+            maxTimeout: timeout,
+            gracePeriod: 30,
+        };
+        this.timeoutManager.startTimer(containerId, timeoutOptions);
+        // Set log TTL now so the key has the correct expiry from the first byte written
+        if (options.dragonflyClient && options.runId) {
+            const logManager = new log_manager_js_1.LogManager(options.dragonflyClient);
+            await logManager.setLogTtl(options.runId, options.userPlan ?? 'free');
+        }
+        // Start container
+        await container.start();
+        console.log(`[DockerManager] Container started with ${timeout}s timeout`);
+        // Wait for container to finish
+        let exitCode = 0;
+        let status = 'success';
+        try {
+            const result = await container.wait();
+            exitCode = result.StatusCode || 0;
+            // Check if it was timed out
+            const timerState = this.timeoutManager.getState(containerId);
+            if (timerState?.status === 'killed') {
+                status = 'killed';
+            }
+            else if (timerState?.status === 'timed_out') {
+                status = 'timed_out';
+            }
+            else if (exitCode !== 0) {
+                status = 'failed';
+            }
+            // Stop the timer
+            this.timeoutManager.stopTimer(containerId);
+        }
+        catch (error) {
+            // Container might have been killed
+            const timerState = this.timeoutManager.getState(containerId);
+            if (timerState?.status === 'killed') {
+                status = 'killed';
+                exitCode = 137; // SIGKILL exit code
+            }
+            else {
+                status = 'failed';
+                exitCode = 1;
+            }
+            this.timeoutManager.stopTimer(containerId);
+        }
+        // Get logs
+        const logs = await this.getLogs(containerId);
+        // Get network stats and track data transfer before removing container
+        await this.trackDataTransfer(container, options);
+        // Remove container
+        try {
+            await container.remove();
+        }
+        catch (e) {
+            // Ignore removal errors
+        }
+        const duration = Math.round((Date.now() - startTime) / 1000);
+        console.log(`[DockerManager] Container finished: ${status} (exit code: ${exitCode}, duration: ${duration}s)`);
+        // Cleanup after run completes
+        const cleanedUp = await this.cleanupAfterRun(options, status);
+        return {
+            containerId,
+            exitCode,
+            logs,
+            status,
+            duration,
+            cleanedUp,
+        };
+    }
+    /**
+     * Cleanup storage after run completes.
+     * - Deletes request queue immediately (no longer needed)
+     * - Uploads logs to S3 for long-term storage
+     * - Updates run metadata with final status
+     */
+    async cleanupAfterRun(options, status) {
+        const result = { requestQueueDeleted: false, logsUploadedToS3: false };
+        if (!options.dragonflyClient || !options.runId) {
+            return result;
+        }
+        try {
+            // Get run metadata to find storage IDs
+            const metadataManager = new run_metadata_js_1.RunMetadataManager(options.dragonflyClient);
+            const metadata = await metadataManager.get(options.runId);
+            if (metadata) {
+                // Delete request queue immediately - it's only useful during execution
+                const rqClient = options.dragonflyClient.requestQueue(metadata.storageIds.requestQueue);
+                await rqClient.delete();
+                result.requestQueueDeleted = true;
+                console.log(`[DockerManager] Deleted request queue: ${metadata.storageIds.requestQueue}`);
+                // Upload logs to S3 for long-term storage
+                const logManager = new log_manager_js_1.LogManager(options.dragonflyClient);
+                const s3Key = await logManager.uploadToS3(options.runId, metadata.userId);
+                result.logsUploadedToS3 = s3Key !== null;
+                // Delete logs from Redis now that they're in S3 — saves RAM.
+                // If S3 upload wasn't configured/failed, TTL set at run start handles expiry.
+                if (result.logsUploadedToS3) {
+                    await logManager.deleteLogs(options.runId);
+                    console.log(`[DockerManager] Deleted Redis logs for run ${options.runId} (persisted to S3)`);
+                }
+                // Update run metadata with final status
+                const runStatus = status === 'success' ? 'success'
+                    : status === 'timed_out' ? 'timed_out'
+                        : status === 'killed' ? 'killed'
+                            : 'failed';
+                await metadataManager.finish(options.runId, runStatus);
+            }
+        }
+        catch (error) {
+            console.error(`[DockerManager] Cleanup error: ${error.message}`);
+        }
+        return result;
+    }
+    /**
+     * Stop a container (SIGTERM)
+     */
+    async stop(containerId, timeoutSeconds = 30) {
+        const container = this.docker.getContainer(containerId);
+        try {
+            await container.stop({ t: timeoutSeconds });
+        }
+        catch (error) {
+            if (error.statusCode !== 304) { // Not modified - already stopped
+                throw error;
+            }
+        }
+    }
+    /**
+     * Kill a container (SIGKILL)
+     */
+    async kill(containerId) {
+        const container = this.docker.getContainer(containerId);
+        try {
+            await container.kill();
+        }
+        catch (error) {
+            if (error.statusCode !== 409) { // Not running
+                throw error;
+            }
+        }
+    }
+    /**
+     * Get logs from a container
+     */
+    async getLogs(containerId) {
+        const container = this.docker.getContainer(containerId);
+        try {
+            const logStream = await container.logs({
+                stdout: true,
+                stderr: true,
+                timestamps: false,
+            });
+            // Docker logs are prefixed with headers, clean them
+            return logStream.toString('utf-8').replace(/\x00\x00\x00[\x00-\xff][\x00-\xff][\x00-\xff]/g, '');
+        }
+        catch (error) {
+            return '';
+        }
+    }
+    /**
+     * Get container status
+     */
+    async getStatus(containerId) {
+        const container = this.docker.getContainer(containerId);
+        try {
+            const info = await container.inspect();
+            return info.State.Status;
+        }
+        catch (error) {
+            return null;
+        }
+    }
+    /**
+     * List all scrapely containers
+     */
+    async listContainers() {
+        const containers = await this.docker.listContainers({
+            all: true,
+            filters: {
+                label: ['scrapely.actor.name'],
+            },
+        });
+        return containers;
+    }
+    /**
+     * Get the timeout manager
+     */
+    getTimeoutManager() {
+        return this.timeoutManager;
+    }
+    /**
+     * Track internal data transfer from container network stats.
+     * Following Apify's model:
+     * - Ingress (rx_bytes) is FREE - scraped content coming in
+     * - Internal transfer is tracked separately in storage clients
+     * - External egress is tracked in API layer when users fetch results
+     */
+    async trackDataTransfer(container, options) {
+        // Ingress is free - no billing for data coming into the platform from internet
+        // Internal transfer (actor ↔ storage) is tracked in storage clients
+        // External egress (platform → user) is tracked in API routes
+        // This method is kept for future use if needed
+    }
+}
+exports.DockerManager = DockerManager;

package/dist/runner/index.d.ts

@@ -0,0 +1,2 @@
+export { DockerManager, RunOptions, RunResult } from './docker-manager.js';
+export { TimeoutManager, TimeoutOptions, TimerState } from './timeout-manager.js';

package/dist/runner/index.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TimeoutManager = exports.DockerManager = void 0;
+var docker_manager_js_1 = require("./docker-manager.js");
+Object.defineProperty(exports, "DockerManager", { enumerable: true, get: function () { return docker_manager_js_1.DockerManager; } });
+var timeout_manager_js_1 = require("./timeout-manager.js");
+Object.defineProperty(exports, "TimeoutManager", { enumerable: true, get: function () { return timeout_manager_js_1.TimeoutManager; } });

package/dist/runner/timeout-manager.d.ts

@@ -0,0 +1,43 @@
+import { EventEmitter } from 'events';
+export interface TimeoutOptions {
+    /** Maximum runtime in seconds */
+    maxTimeout: number;
+    /** Grace period after SIGTERM before SIGKILL (default: 30s) */
+    gracePeriod?: number;
+}
+export interface TimerState {
+    containerId: string;
+    startedAt: Date;
+    timeoutAt: Date;
+    status: 'running' | 'stopping' | 'timed_out' | 'completed' | 'killed';
+    timeoutTimer?: NodeJS.Timeout;
+    killTimer?: NodeJS.Timeout;
+}
+export declare class TimeoutManager extends EventEmitter {
+    private timers;
+    private defaultGracePeriod;
+    /**
+     * Start a timeout timer for a container
+     */
+    startTimer(containerId: string, options: TimeoutOptions): TimerState;
+    /**
+     * Stop a timer (container completed successfully)
+     */
+    stopTimer(containerId: string): void;
+    /**
+     * Get timer state for a container
+     */
+    getState(containerId: string): TimerState | undefined;
+    /**
+     * Get remaining time in seconds for a container
+     */
+    getRemainingTime(containerId: string): number | undefined;
+    /**
+     * Get all active timers
+     */
+    getActiveTimers(): TimerState[];
+    /**
+     * Clear all timers
+     */
+    clearAll(): void;
+}

package/dist/runner/timeout-manager.js

@@ -0,0 +1,90 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TimeoutManager = void 0;
+const events_1 = require("events");
+class TimeoutManager extends events_1.EventEmitter {
+    timers = new Map();
+    defaultGracePeriod = 30; // seconds
+    /**
+     * Start a timeout timer for a container
+     */
+    startTimer(containerId, options) {
+        // Stop existing timer if any
+        this.stopTimer(containerId);
+        const startedAt = new Date();
+        const timeoutAt = new Date(startedAt.getTime() + options.maxTimeout * 1000);
+        const gracePeriod = options.gracePeriod ?? this.defaultGracePeriod;
+        const state = {
+            containerId,
+            startedAt,
+            timeoutAt,
+            status: 'running',
+        };
+        // Set up timeout timer
+        state.timeoutTimer = setTimeout(async () => {
+            this.emit('timeout', { containerId, state });
+            state.status = 'timed_out';
+            this.emit('stopping', { containerId, gracePeriod });
+            // Set up kill timer after timeout
+            state.killTimer = setTimeout(() => {
+                state.status = 'killed';
+                this.emit('kill', { containerId });
+                this.timers.delete(containerId);
+            }, gracePeriod * 1000);
+        }, options.maxTimeout * 1000);
+        this.timers.set(containerId, state);
+        this.emit('started', { containerId, timeoutAt });
+        return state;
+    }
+    /**
+     * Stop a timer (container completed successfully)
+     */
+    stopTimer(containerId) {
+        const state = this.timers.get(containerId);
+        if (!state)
+            return;
+        // Clear timers
+        if (state.timeoutTimer) {
+            clearTimeout(state.timeoutTimer);
+        }
+        if (state.killTimer) {
+            clearTimeout(state.killTimer);
+        }
+        if (state.status === 'running') {
+            state.status = 'completed';
+            this.emit('completed', { containerId, state });
+        }
+        this.timers.delete(containerId);
+    }
+    /**
+     * Get timer state for a container
+     */
+    getState(containerId) {
+        return this.timers.get(containerId);
+    }
+    /**
+     * Get remaining time in seconds for a container
+     */
+    getRemainingTime(containerId) {
+        const state = this.timers.get(containerId);
+        if (!state || state.status !== 'running')
+            return undefined;
+        const remaining = Math.max(0, (state.timeoutAt.getTime() - Date.now()) / 1000);
+        return remaining;
+    }
+    /**
+     * Get all active timers
+     */
+    getActiveTimers() {
+        return Array.from(this.timers.values()).filter(s => s.status === 'running');
+    }
+    /**
+     * Clear all timers
+     */
+    clearAll() {
+        for (const [containerId] of this.timers) {
+            this.stopTimer(containerId);
+        }
+    }
+}
+exports.TimeoutManager = TimeoutManager;

package/dist/scrapely-proxy-configuration.d.ts

@@ -0,0 +1,99 @@
+import { ProxyConfiguration as CrawleeProxyConfiguration } from '@crawlee/core';
+import { Configuration } from './configuration.js';
+import { type PlatformProxyEnv, type ProxyConfigurationOptions, type ProxyInfo } from './proxy-configuration.js';
+/** Options accepted by newUrl()/newProxyInfo() (second parameter). */
+type TieredProxyOptions = Parameters<CrawleeProxyConfiguration['newUrl']>[1];
+/** Returns true for platform-proxy sentinel values ('auto', 'http://auto'). */
+export declare function isPlatformProxySentinel(url: string | null | undefined): boolean;
+/**
+ * Validates a proxy session ID the same way the Apify SDK does: at most 50
+ * characters, and only `0-9`, `a-z`, `A-Z`, `.`, `_` and `~`. Throws otherwise.
+ */
+export declare function validateProxySessionId(sessionId: string): string;
+/**
+ * Resolves and validates the country/subdivision options (Apify parity).
+ * Country must be a two-letter ISO code; subdivision an ISO 3166-2 code
+ * (1-3 uppercase alphanumeric characters) and requires a country. Returns
+ * the normalized values used by the username grammar.
+ */
+export declare function resolveCountryOptions(options: ProxyConfigurationOptions): {
+    countryCode?: string;
+    subdivisionCode?: string;
+};
+/** Status payload returned by the gateway status endpoint (Apify format). */
+export interface ProxyStatus {
+    connected: boolean;
+    connectionError: string;
+    isManInTheMiddle: boolean;
+}
+/**
+ * Build a platform proxy URL using the Apify username grammar:
+ *   http://GROUPS-G1+G2,country-US,session-a1b2c3:PASSWORD@HOST:PORT
+ *
+ * `groups` (joined with `+`), `country` and `session` fragments are
+ * comma-joined. When the server injected a subaccount username
+ * (SCRAPELY_PROXY_USERNAME / APIFY_PROXY_USERNAME) it is prepended so the
+ * gateway can attribute traffic to the right Scrapely tenant; without it the
+ * username falls back to Apify's `auto` when no fragments apply. The password
+ * and gateway host/port come from the environment credentials injected by the
+ * server.
+ */
+export declare function buildPlatformProxyUrl(options: ProxyConfigurationOptions, env: PlatformProxyEnv, sessionId?: string): string;
+/** True when any proxy URL list reserves a platform slot via the 'auto' sentinel. */
+export declare function hasPlatformSentinel(options: ProxyConfigurationOptions): boolean;
+/**
+ * Apify-compatible ProxyConfiguration.
+ *
+ * Modes:
+ *  - Platform proxy: when the server injected APIFY_PROXY_* or SCRAPELY_PROXY_*
+ *    credentials (Evomi-backed) and no custom proxies were requested, the
+ *    configuration routes through the platform gateway using the Apify
+ *    username grammar (groups/country/session fragments).
+ *  - BYOP: custom proxyUrls / newUrlFunction / tieredProxyUrls are passed
+ *    through to Crawlee unchanged.
+ *  - Mixed: the 'auto' (or 'http://auto') sentinel inside proxyUrls /
+ *    tieredProxyUrls / tieredProxyConfig[].proxyUrls reserves a slot for the
+ *    platform proxy alongside BYOP entries. The sentinel is translated to a
+ *    concrete platform URL lazily on every newUrl() call, so Crawlee's
+ *    rotation and session-stickiness semantics apply to the mix unchanged.
+ */
+export declare class ProxyConfiguration extends CrawleeProxyConfiguration {
+    readonly config: Configuration;
+    private readonly platformEnv;
+    private readonly platformOptions;
+    /** Platform options to use when translating each per-tier placeholder. */
+    private readonly tierPlaceholderOptions;
+    constructor(options?: ProxyConfigurationOptions, config?: Configuration);
+    /**
+     * Returns a new proxy URL. Translates the internal platform-proxy
+     * placeholder to a concrete platform URL (applying groups/countryCode and
+     * session stickiness) whenever Crawlee's rotation lands on it; BYOP
+     * entries pass through unchanged. Session IDs are validated like in the
+     * Apify SDK (max 50 chars, only `0-9 a-z A-Z . _ ~`).
+     */
+    newUrl(sessionId?: string | number, options?: TieredProxyOptions): Promise<string | undefined>;
+    /**
+     * Returns a new ProxyInfo. When the resolved URL points at the platform
+     * gateway, `groups`, `countryCode` and `subdivisionCode` are attached
+     * (Apify parity).
+     */
+    newProxyInfo(sessionId?: string | number, options?: TieredProxyOptions): Promise<ProxyInfo | undefined>;
+    /**
+     * Fetches the gateway status payload (Apify format) by sending
+     * `GET {statusUrl}/?format=json` *through the platform gateway*, so the
+     * proxy handshake itself authenticates the run's credentials. Retries
+     * connection errors (Apify parity: 4s timeout, 2 attempts); returns
+     * undefined when the endpoint stays unreachable.
+     */
+    private fetchProxyStatus;
+    /**
+     * Verifies the proxy configuration. For platform proxy mode with an
+     * injected status URL, performs the Apify-format access check through the
+     * gateway (`GET {statusUrl}/?format=json`, best-effort, never fatal);
+     * BYOP mode only verifies that a test URL can be generated.
+     */
+    initialize(options?: {
+        checkAccess?: boolean;
+    }): Promise<boolean>;
+}
+export {};