scrapely 1.2.32 → 1.2.33

package/dist/solver.js

@@ -31,7 +31,7 @@ exports.solve = solve;
 exports.reportSolveOutcome = reportSolveOutcome;
 exports.clearSolverStatusCache = clearSolverStatusCache;
 exports.DEGRADED_STATUS_MESSAGE = 'Site protection temporarily unavailable, results may be incomplete';
-const COMPAT_SUFFIXES = ['/compat/v1', '/compat/xhrdev'];
+const COMPAT_SUFFIXES = ['/compat/v1'];
 const STATUS_TTL_MS = 15_000;
 const MAX_BODY = 4 * 1024 * 1024;
 const UNAVAILABLE_CODES = new Set(['vendor_degraded', 'vendor_down', 'provider_unavailable', 'vendor_unsupported', 'solver_not_enabled']);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.32",
+    "version": "1.2.33",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",

package/dist/runner/docker-manager.d.ts

@@ -1,88 +0,0 @@
-import Docker from 'dockerode';
-import { TimeoutManager } from './timeout-manager.js';
-import { DragonflyClient } from '../storage/dragonfly-client.js';
-import type { UserPlan } from '../storage/ttl-manager.js';
-export interface RunOptions {
-    /** Path to the actor package directory */
-    packagePath: string;
-    /** Path to the .env file */
-    envPath?: string;
-    /** Environment variables to pass to container */
-    env?: Record<string, string>;
-    /** Maximum memory in MB (default: from manifest or 512) */
-    memoryMb?: number;
-    /** Maximum timeout in seconds (default: from manifest or 1800) */
-    timeout?: number;
-    /** Run ID for tracking */
-    runId?: string;
-    /** User ID */
-    userId?: string;
-    /** User plan for TTL */
-    userPlan?: UserPlan;
-    /** Dragonfly client for storage cleanup */
-    dragonflyClient?: DragonflyClient;
-}
-export interface RunResult {
-    containerId: string;
-    exitCode: number;
-    logs: string;
-    status: 'success' | 'failed' | 'timed_out' | 'killed';
-    duration: number;
-    /** Storage IDs that were cleaned up */
-    cleanedUp?: {
-        requestQueueDeleted: boolean;
-        logsUploadedToS3: boolean;
-    };
-}
-export declare class DockerManager {
-    private docker;
-    private timeoutManager;
-    private imageName;
-    constructor(options?: {
-        socketPath?: string;
-        imageName?: string;
-    });
-    /**
-     * Run an actor package in a container
-     */
-    run(options: RunOptions): Promise<RunResult>;
-    /**
-     * Cleanup storage after run completes.
-     * - Deletes request queue immediately (no longer needed)
-     * - Uploads logs to S3 for long-term storage
-     * - Updates run metadata with final status
-     */
-    private cleanupAfterRun;
-    /**
-     * Stop a container (SIGTERM)
-     */
-    stop(containerId: string, timeoutSeconds?: number): Promise<void>;
-    /**
-     * Kill a container (SIGKILL)
-     */
-    kill(containerId: string): Promise<void>;
-    /**
-     * Get logs from a container
-     */
-    getLogs(containerId: string): Promise<string>;
-    /**
-     * Get container status
-     */
-    getStatus(containerId: string): Promise<string | null>;
-    /**
-     * List all scrapely containers
-     */
-    listContainers(): Promise<Docker.ContainerInfo[]>;
-    /**
-     * Get the timeout manager
-     */
-    getTimeoutManager(): TimeoutManager;
-    /**
-     * Track internal data transfer from container network stats.
-     * Following Apify's model:
-     * - Ingress (rx_bytes) is FREE - scraped content coming in
-     * - Internal transfer is tracked separately in storage clients
-     * - External egress is tracked in API layer when users fetch results
-     */
-    private trackDataTransfer;
-}

package/dist/runner/docker-manager.js

@@ -1,283 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DockerManager = void 0;
-const dockerode_1 = __importDefault(require("dockerode"));
-const timeout_manager_js_1 = require("./timeout-manager.js");
-const validator_js_1 = require("../validator.js");
-const path_1 = __importDefault(require("path"));
-const run_metadata_js_1 = require("../storage/run-metadata.js");
-const log_manager_js_1 = require("../storage/log-manager.js");
-class DockerManager {
-    docker;
-    timeoutManager;
-    imageName;
-    constructor(options) {
-        this.docker = new dockerode_1.default({
-            socketPath: options?.socketPath || '/var/run/docker.sock',
-        });
-        this.timeoutManager = new timeout_manager_js_1.TimeoutManager();
-        this.imageName = options?.imageName || 'scrapely-runner:latest';
-        // Set up timeout event handlers
-        this.timeoutManager.on('timeout', async ({ containerId }) => {
-            console.log(`[DockerManager] Container ${containerId} timed out, sending SIGTERM...`);
-            await this.stop(containerId, 30);
-        });
-        this.timeoutManager.on('kill', async ({ containerId }) => {
-            console.log(`[DockerManager] Container ${containerId} grace period expired, sending SIGKILL...`);
-            await this.kill(containerId);
-        });
-    }
-    /**
-     * Run an actor package in a container
-     */
-    async run(options) {
-        // Validate package
-        const validation = (0, validator_js_1.validatePackage)(options.packagePath);
-        if (!validation.valid) {
-            throw new Error(`Invalid package: ${validation.errors.join(', ')}`);
-        }
-        const manifest = validation.manifest;
-        const startTime = Date.now();
-        const memoryMb = options.memoryMb || manifest.maxMemory || 512;
-        const timeout = options.timeout || manifest.maxTimeout || 1800;
-        // Build environment variables
-        const envVars = [];
-        if (options.env) {
-            for (const [key, value] of Object.entries(options.env)) {
-                envVars.push(`${key}=${value}`);
-            }
-        }
-        // Build volume mounts
-        const binds = [
-            `${path_1.default.resolve(options.packagePath)}:/actor/package:ro`,
-        ];
-        if (options.envPath) {
-            binds.push(`${path_1.default.resolve(options.envPath)}:/actor/.env:ro`);
-        }
-        // Create container
-        console.log(`[DockerManager] Creating container for ${manifest.name}@${manifest.version}...`);
-        const container = await this.docker.createContainer({
-            Image: this.imageName,
-            Env: envVars,
-            HostConfig: {
-                Binds: binds,
-                Memory: memoryMb * 1024 * 1024, // Convert MB to bytes
-                AutoRemove: false, // We'll remove it manually after getting logs
-            },
-            Labels: {
-                'scrapely.actor.name': manifest.name,
-                'scrapely.actor.version': manifest.version,
-            },
-        });
-        const containerId = container.id;
-        console.log(`[DockerManager] Container created: ${containerId.substring(0, 12)}`);
-        // Start timeout timer
-        const timeoutOptions = {
-            maxTimeout: timeout,
-            gracePeriod: 30,
-        };
-        this.timeoutManager.startTimer(containerId, timeoutOptions);
-        // Set log TTL now so the key has the correct expiry from the first byte written
-        if (options.dragonflyClient && options.runId) {
-            const logManager = new log_manager_js_1.LogManager(options.dragonflyClient);
-            await logManager.setLogTtl(options.runId, options.userPlan ?? 'free');
-        }
-        // Start container
-        await container.start();
-        console.log(`[DockerManager] Container started with ${timeout}s timeout`);
-        // Wait for container to finish
-        let exitCode = 0;
-        let status = 'success';
-        try {
-            const result = await container.wait();
-            exitCode = result.StatusCode || 0;
-            // Check if it was timed out
-            const timerState = this.timeoutManager.getState(containerId);
-            if (timerState?.status === 'killed') {
-                status = 'killed';
-            }
-            else if (timerState?.status === 'timed_out') {
-                status = 'timed_out';
-            }
-            else if (exitCode !== 0) {
-                status = 'failed';
-            }
-            // Stop the timer
-            this.timeoutManager.stopTimer(containerId);
-        }
-        catch (error) {
-            // Container might have been killed
-            const timerState = this.timeoutManager.getState(containerId);
-            if (timerState?.status === 'killed') {
-                status = 'killed';
-                exitCode = 137; // SIGKILL exit code
-            }
-            else {
-                status = 'failed';
-                exitCode = 1;
-            }
-            this.timeoutManager.stopTimer(containerId);
-        }
-        // Get logs
-        const logs = await this.getLogs(containerId);
-        // Get network stats and track data transfer before removing container
-        await this.trackDataTransfer(container, options);
-        // Remove container
-        try {
-            await container.remove();
-        }
-        catch (e) {
-            // Ignore removal errors
-        }
-        const duration = Math.round((Date.now() - startTime) / 1000);
-        console.log(`[DockerManager] Container finished: ${status} (exit code: ${exitCode}, duration: ${duration}s)`);
-        // Cleanup after run completes
-        const cleanedUp = await this.cleanupAfterRun(options, status);
-        return {
-            containerId,
-            exitCode,
-            logs,
-            status,
-            duration,
-            cleanedUp,
-        };
-    }
-    /**
-     * Cleanup storage after run completes.
-     * - Deletes request queue immediately (no longer needed)
-     * - Uploads logs to S3 for long-term storage
-     * - Updates run metadata with final status
-     */
-    async cleanupAfterRun(options, status) {
-        const result = { requestQueueDeleted: false, logsUploadedToS3: false };
-        if (!options.dragonflyClient || !options.runId) {
-            return result;
-        }
-        try {
-            // Get run metadata to find storage IDs
-            const metadataManager = new run_metadata_js_1.RunMetadataManager(options.dragonflyClient);
-            const metadata = await metadataManager.get(options.runId);
-            if (metadata) {
-                // Delete request queue immediately - it's only useful during execution
-                const rqClient = options.dragonflyClient.requestQueue(metadata.storageIds.requestQueue);
-                await rqClient.delete();
-                result.requestQueueDeleted = true;
-                console.log(`[DockerManager] Deleted request queue: ${metadata.storageIds.requestQueue}`);
-                // Upload logs to S3 for long-term storage
-                const logManager = new log_manager_js_1.LogManager(options.dragonflyClient);
-                const s3Key = await logManager.uploadToS3(options.runId, metadata.userId);
-                result.logsUploadedToS3 = s3Key !== null;
-                // Delete logs from Redis now that they're in S3 — saves RAM.
-                // If S3 upload wasn't configured/failed, TTL set at run start handles expiry.
-                if (result.logsUploadedToS3) {
-                    await logManager.deleteLogs(options.runId);
-                    console.log(`[DockerManager] Deleted Redis logs for run ${options.runId} (persisted to S3)`);
-                }
-                // Update run metadata with final status
-                const runStatus = status === 'success' ? 'success'
-                    : status === 'timed_out' ? 'timed_out'
-                        : status === 'killed' ? 'killed'
-                            : 'failed';
-                await metadataManager.finish(options.runId, runStatus);
-            }
-        }
-        catch (error) {
-            console.error(`[DockerManager] Cleanup error: ${error.message}`);
-        }
-        return result;
-    }
-    /**
-     * Stop a container (SIGTERM)
-     */
-    async stop(containerId, timeoutSeconds = 30) {
-        const container = this.docker.getContainer(containerId);
-        try {
-            await container.stop({ t: timeoutSeconds });
-        }
-        catch (error) {
-            if (error.statusCode !== 304) { // Not modified - already stopped
-                throw error;
-            }
-        }
-    }
-    /**
-     * Kill a container (SIGKILL)
-     */
-    async kill(containerId) {
-        const container = this.docker.getContainer(containerId);
-        try {
-            await container.kill();
-        }
-        catch (error) {
-            if (error.statusCode !== 409) { // Not running
-                throw error;
-            }
-        }
-    }
-    /**
-     * Get logs from a container
-     */
-    async getLogs(containerId) {
-        const container = this.docker.getContainer(containerId);
-        try {
-            const logStream = await container.logs({
-                stdout: true,
-                stderr: true,
-                timestamps: false,
-            });
-            // Docker logs are prefixed with headers, clean them
-            return logStream.toString('utf-8').replace(/\x00\x00\x00[\x00-\xff][\x00-\xff][\x00-\xff]/g, '');
-        }
-        catch (error) {
-            return '';
-        }
-    }
-    /**
-     * Get container status
-     */
-    async getStatus(containerId) {
-        const container = this.docker.getContainer(containerId);
-        try {
-            const info = await container.inspect();
-            return info.State.Status;
-        }
-        catch (error) {
-            return null;
-        }
-    }
-    /**
-     * List all scrapely containers
-     */
-    async listContainers() {
-        const containers = await this.docker.listContainers({
-            all: true,
-            filters: {
-                label: ['scrapely.actor.name'],
-            },
-        });
-        return containers;
-    }
-    /**
-     * Get the timeout manager
-     */
-    getTimeoutManager() {
-        return this.timeoutManager;
-    }
-    /**
-     * Track internal data transfer from container network stats.
-     * Following Apify's model:
-     * - Ingress (rx_bytes) is FREE - scraped content coming in
-     * - Internal transfer is tracked separately in storage clients
-     * - External egress is tracked in API layer when users fetch results
-     */
-    async trackDataTransfer(container, options) {
-        // Ingress is free - no billing for data coming into the platform from internet
-        // Internal transfer (actor ↔ storage) is tracked in storage clients
-        // External egress (platform → user) is tracked in API routes
-        // This method is kept for future use if needed
-    }
-}
-exports.DockerManager = DockerManager;

package/dist/runner/index.d.ts

@@ -1,2 +0,0 @@
-export { DockerManager, RunOptions, RunResult } from './docker-manager.js';
-export { TimeoutManager, TimeoutOptions, TimerState } from './timeout-manager.js';

package/dist/runner/index.js

@@ -1,7 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TimeoutManager = exports.DockerManager = void 0;
-var docker_manager_js_1 = require("./docker-manager.js");
-Object.defineProperty(exports, "DockerManager", { enumerable: true, get: function () { return docker_manager_js_1.DockerManager; } });
-var timeout_manager_js_1 = require("./timeout-manager.js");
-Object.defineProperty(exports, "TimeoutManager", { enumerable: true, get: function () { return timeout_manager_js_1.TimeoutManager; } });

package/dist/runner/timeout-manager.d.ts

@@ -1,43 +0,0 @@
-import { EventEmitter } from 'events';
-export interface TimeoutOptions {
-    /** Maximum runtime in seconds */
-    maxTimeout: number;
-    /** Grace period after SIGTERM before SIGKILL (default: 30s) */
-    gracePeriod?: number;
-}
-export interface TimerState {
-    containerId: string;
-    startedAt: Date;
-    timeoutAt: Date;
-    status: 'running' | 'stopping' | 'timed_out' | 'completed' | 'killed';
-    timeoutTimer?: NodeJS.Timeout;
-    killTimer?: NodeJS.Timeout;
-}
-export declare class TimeoutManager extends EventEmitter {
-    private timers;
-    private defaultGracePeriod;
-    /**
-     * Start a timeout timer for a container
-     */
-    startTimer(containerId: string, options: TimeoutOptions): TimerState;
-    /**
-     * Stop a timer (container completed successfully)
-     */
-    stopTimer(containerId: string): void;
-    /**
-     * Get timer state for a container
-     */
-    getState(containerId: string): TimerState | undefined;
-    /**
-     * Get remaining time in seconds for a container
-     */
-    getRemainingTime(containerId: string): number | undefined;
-    /**
-     * Get all active timers
-     */
-    getActiveTimers(): TimerState[];
-    /**
-     * Clear all timers
-     */
-    clearAll(): void;
-}

package/dist/runner/timeout-manager.js

@@ -1,90 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TimeoutManager = void 0;
-const events_1 = require("events");
-class TimeoutManager extends events_1.EventEmitter {
-    timers = new Map();
-    defaultGracePeriod = 30; // seconds
-    /**
-     * Start a timeout timer for a container
-     */
-    startTimer(containerId, options) {
-        // Stop existing timer if any
-        this.stopTimer(containerId);
-        const startedAt = new Date();
-        const timeoutAt = new Date(startedAt.getTime() + options.maxTimeout * 1000);
-        const gracePeriod = options.gracePeriod ?? this.defaultGracePeriod;
-        const state = {
-            containerId,
-            startedAt,
-            timeoutAt,
-            status: 'running',
-        };
-        // Set up timeout timer
-        state.timeoutTimer = setTimeout(async () => {
-            this.emit('timeout', { containerId, state });
-            state.status = 'timed_out';
-            this.emit('stopping', { containerId, gracePeriod });
-            // Set up kill timer after timeout
-            state.killTimer = setTimeout(() => {
-                state.status = 'killed';
-                this.emit('kill', { containerId });
-                this.timers.delete(containerId);
-            }, gracePeriod * 1000);
-        }, options.maxTimeout * 1000);
-        this.timers.set(containerId, state);
-        this.emit('started', { containerId, timeoutAt });
-        return state;
-    }
-    /**
-     * Stop a timer (container completed successfully)
-     */
-    stopTimer(containerId) {
-        const state = this.timers.get(containerId);
-        if (!state)
-            return;
-        // Clear timers
-        if (state.timeoutTimer) {
-            clearTimeout(state.timeoutTimer);
-        }
-        if (state.killTimer) {
-            clearTimeout(state.killTimer);
-        }
-        if (state.status === 'running') {
-            state.status = 'completed';
-            this.emit('completed', { containerId, state });
-        }
-        this.timers.delete(containerId);
-    }
-    /**
-     * Get timer state for a container
-     */
-    getState(containerId) {
-        return this.timers.get(containerId);
-    }
-    /**
-     * Get remaining time in seconds for a container
-     */
-    getRemainingTime(containerId) {
-        const state = this.timers.get(containerId);
-        if (!state || state.status !== 'running')
-            return undefined;
-        const remaining = Math.max(0, (state.timeoutAt.getTime() - Date.now()) / 1000);
-        return remaining;
-    }
-    /**
-     * Get all active timers
-     */
-    getActiveTimers() {
-        return Array.from(this.timers.values()).filter(s => s.status === 'running');
-    }
-    /**
-     * Clear all timers
-     */
-    clearAll() {
-        for (const [containerId] of this.timers) {
-            this.stopTimer(containerId);
-        }
-    }
-}
-exports.TimeoutManager = TimeoutManager;