scrapely 1.2.3 → 1.2.5

package/dist/proxy-configuration.d.ts

@@ -0,0 +1,118 @@
+import type { ProxyConfigurationOptions as CoreProxyConfigurationOptions, ProxyInfo as CoreProxyInfo } from '@crawlee/core';
+import { ProxyConfiguration } from './scrapely-proxy-configuration.js';
+/**
+ * Proxy info enriched with the platform-proxy (Apify-compatible) username
+ * fields. Mirrors the Apify SDK's `ProxyInfo` extension of Crawlee's type.
+ */
+export interface ProxyInfo extends CoreProxyInfo {
+    /** Proxy groups used by the platform proxy (e.g. `['RESIDENTIAL']`). */
+    groups?: string[];
+    /** Two-letter ISO country code requested from the platform proxy. */
+    countryCode?: string;
+    /** ISO 3166-2 subdivision code requested from the platform proxy. */
+    subdivisionCode?: string;
+}
+/**
+ * Options for creating a proxy configuration.
+ *
+ * Modes supported:
+ *  - Platform proxy (default on Scrapely runs): when the server injects
+ *    APIFY_PROXY_* or SCRAPELY_PROXY_* credentials, groups/countryCode/password
+ *    build Apify-style gateway usernames automatically.
+ *  - Bring-your-own-proxy (BYOP): custom proxyUrls / newUrlFunction /
+ *    tieredProxyUrls always win over the platform proxy.
+ *  - Mixed: the 'auto' (or 'http://auto') sentinel inside proxyUrls /
+ *    tieredProxyUrls / tieredProxyConfig[].proxyUrls reserves a slot for the
+ *    platform proxy alongside BYOP entries (Apify-compatible).
+ */
+export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions {
+    /**
+     * Multiple different ProxyConfigurationOptions stratified into tiers (Apify-compatible).
+     * Tiers with `proxyUrls` map directly to tieredProxyUrls; tiers without
+     * URLs expand their platform options (groups/countryCode/password) to a
+     * concrete platform proxy URL, mirroring Apify's `_generateTieredProxyUrls`.
+     */
+    tieredProxyConfig?: Omit<ProxyConfigurationOptions, 'tieredProxyConfig' | 'tieredProxyUrls'>[];
+    /**
+     * Platform proxy groups (Apify-compatible). Effective in platform proxy
+     * mode; ignored for BYOP configurations.
+     */
+    groups?: string[];
+    /**
+     * Platform proxy country code (Apify-compatible). Effective in platform
+     * proxy mode; ignored for BYOP configurations.
+     */
+    countryCode?: string;
+    /**
+     * Platform proxy subdivision code, e.g. US state (Apify-compatible).
+     * Requires `countryCode`. Effective in platform proxy mode; ignored for
+     * BYOP configurations. Emitted as a merged `country-XX_YY` username
+     * fragment (ISO 3166-2).
+     */
+    subdivisionCode?: string;
+    /**
+     * Input-schema alias for groups. Normalized to groups.
+     */
+    apifyProxyGroups?: string[];
+    /**
+     * Input-schema alias for subdivisionCode. Normalized to subdivisionCode.
+     */
+    apifyProxySubdivision?: string;
+    /**
+     * Input-schema alias for countryCode. Normalized to countryCode.
+     */
+    apifyProxyCountry?: string;
+    /**
+     * Platform proxy password. Defaults to the password injected by the
+     * server (APIFY_PROXY_PASSWORD / SCRAPELY_PROXY_PASSWORD).
+     */
+    password?: string;
+}
+/**
+ * Options for Actor.createProxyConfiguration().
+ */
+export interface CreateProxyConfigurationOptions extends ProxyConfigurationOptions {
+    /**
+     * For compatibility with existing Actor Input UI (Input Schema).
+     * When set to `false`, this method returns `undefined`.
+     */
+    useApifyProxy?: boolean;
+    /**
+     * Whether to verify the proxy configuration by testing a connection.
+     * @default true
+     */
+    checkAccess?: boolean;
+}
+/**
+ * Platform proxy connection details injected into run containers by the
+ * Scrapely server (backed by Evomi). SCRAPELY_* vars take precedence over the
+ * Apify-compatible aliases so Scrapely-first deployments stay unambiguous.
+ */
+export interface PlatformProxyEnv {
+    hostname: string;
+    port: number;
+    password: string;
+    /**
+     * Subaccount username injected by the server. When present it is prefixed
+     * to every generated gateway username (groups/country/session fragments
+     * are appended after it) so the gateway attributes traffic to the right
+     * Scrapely tenant.
+     */
+    username?: string;
+    statusUrl?: string;
+}
+/**
+ * Read platform proxy credentials from the environment.
+ * Returns null when the run has no platform proxy (BYOP / no-proxy mode).
+ */
+export declare function getPlatformProxyEnv(): PlatformProxyEnv | null;
+/**
+ * Creates a proxy configuration for use with crawlers.
+ *
+ * The platform proxy (Evomi-backed) is injected into runs via the SCRAPELY_PROXY
+ * and APIFY_PROXY environment variables. Apify Proxy options (groups, countryCode, ...)
+ * build gateway usernames in platform mode; proxyUrls, newUrlFunction and
+ * tieredProxyUrls configure bring-your-own proxies instead.
+ */
+export declare function createProxyConfiguration(options?: CreateProxyConfigurationOptions): Promise<ProxyConfiguration | undefined>;
+export type { ProxyConfigurationFunction } from '@crawlee/core';

package/dist/proxy-configuration.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPlatformProxyEnv = getPlatformProxyEnv;
+exports.createProxyConfiguration = createProxyConfiguration;
+const scrapely_proxy_configuration_js_1 = require("./scrapely-proxy-configuration.js");
+/**
+ * Read platform proxy credentials from the environment.
+ * Returns null when the run has no platform proxy (BYOP / no-proxy mode).
+ */
+function getPlatformProxyEnv() {
+    const hostname = process.env.SCRAPELY_PROXY_HOSTNAME || process.env.APIFY_PROXY_HOSTNAME;
+    const portStr = process.env.SCRAPELY_PROXY_PORT || process.env.APIFY_PROXY_PORT;
+    const password = process.env.SCRAPELY_PROXY_PASSWORD || process.env.APIFY_PROXY_PASSWORD;
+    if (!hostname || !password)
+        return null;
+    const parsedPort = parseInt(portStr || '', 10);
+    const username = process.env.SCRAPELY_PROXY_USERNAME || process.env.APIFY_PROXY_USERNAME;
+    return {
+        hostname,
+        // Same bounds as the Python SDK (_platform_env): invalid/out-of-range
+        // ports fall back to the default gateway port instead of producing a
+        // broken URL.
+        port: Number.isFinite(parsedPort) && parsedPort > 0 && parsedPort < 65536 ? parsedPort : 8000,
+        password,
+        username: username || undefined,
+        statusUrl: process.env.SCRAPELY_PROXY_STATUS_URL || process.env.APIFY_PROXY_STATUS_URL || undefined,
+    };
+}
+const BYOP_WARNING = '[Scrapely] No proxy configured. Scrapely injects platform proxy credentials (APIFY_PROXY_*/SCRAPELY_PROXY_*) into runs when your account has proxy access. ' +
+    'Locally, either add proxyUrls to your actor input, or copy your proxy credentials from the Scrapely console (Proxy page) into the ' +
+    'SCRAPELY_PROXY_HOSTNAME, SCRAPELY_PROXY_PORT, SCRAPELY_PROXY_USERNAME and SCRAPELY_PROXY_PASSWORD environment variables. ' +
+    'Using the platform proxy from outside Scrapely requires an active plan or a prepaid (PAYG) balance of at least $1; ' +
+    'free accounts get run-scoped credentials injected into their on-platform runs only.';
+function hasCustomProxyOptions(options) {
+    return !!((options.proxyUrls && options.proxyUrls.length > 0) ||
+        typeof options.newUrlFunction === 'function' ||
+        (options.tieredProxyUrls && options.tieredProxyUrls.length > 0) ||
+        (options.tieredProxyConfig && options.tieredProxyConfig.length > 0));
+}
+function normalizeProxyInput(options) {
+    const normalized = { ...options };
+    if (normalized.apifyProxyGroups && !normalized.groups) {
+        normalized.groups = normalized.apifyProxyGroups;
+    }
+    if (normalized.apifyProxyCountry && !normalized.countryCode) {
+        normalized.countryCode = normalized.apifyProxyCountry;
+    }
+    if (normalized.apifyProxySubdivision && !normalized.subdivisionCode) {
+        normalized.subdivisionCode = normalized.apifyProxySubdivision;
+    }
+    return normalized;
+}
+/**
+ * Creates a proxy configuration for use with crawlers.
+ *
+ * The platform proxy (Evomi-backed) is injected into runs via the SCRAPELY_PROXY
+ * and APIFY_PROXY environment variables. Apify Proxy options (groups, countryCode, ...)
+ * build gateway usernames in platform mode; proxyUrls, newUrlFunction and
+ * tieredProxyUrls configure bring-your-own proxies instead.
+ */
+async function createProxyConfiguration(options = {}) {
+    const normalized = normalizeProxyInput(options);
+    const { useApifyProxy, checkAccess = true, ...proxyOptions } = normalized;
+    // Apify parity: `useApifyProxy: false` explicitly disables the PLATFORM proxy
+    // only — custom proxies from the same input must still be used. The standard
+    // "Custom proxy URLs" input shape is `{ useApifyProxy: false, proxyUrls: [...] }`;
+    // returning undefined for it would silently drop the user's own proxies and
+    // crawl on their real IPs. We only bail when NO custom proxy mode is given —
+    // proxyUrls, tieredProxyUrls, tieredProxyConfig and newUrlFunction all count
+    // (Python SDK Actor.create_proxy_configuration parity).
+    const dontUseApifyProxy = useApifyProxy === false;
+    const dontUseCustomProxies = !hasCustomProxyOptions(proxyOptions);
+    if (dontUseApifyProxy && dontUseCustomProxies) {
+        return undefined;
+    }
+    if (!hasCustomProxyOptions(proxyOptions)) {
+        // Platform proxy mode: credentials were injected into the environment
+        // by the Scrapely server (Evomi-backed). groups/countryCode become
+        // meaningful gateway username fragments instead of being ignored.
+        if (!getPlatformProxyEnv()) {
+            console.warn(BYOP_WARNING);
+            return undefined;
+        }
+    }
+    const proxyConfiguration = new scrapely_proxy_configuration_js_1.ProxyConfiguration(proxyOptions);
+    await proxyConfiguration.initialize({ checkAccess });
+    return proxyConfiguration;
+}

package/dist/request-list.d.ts

@@ -0,0 +1,57 @@
+/**
+ * ScrapelyRequestList - parses the standard Scrapely/Apify `requestListSources` input format.
+ *
+ * Parity port of the Python SDK's `scrapely.request_loaders.ScrapelyRequestList`.
+ * It supports two kinds of entries:
+ *
+ * - **Direct URLs** - entries with a `url` key are converted to requests directly.
+ * - **Remote URL lists** - entries with a `requestsFromUrl` key point to a remote resource
+ *   (e.g. a plain-text file). The resource is fetched and all URLs found in the response
+ *   body are extracted and converted to requests.
+ *
+ * Both kinds of entries can optionally specify `method`, `payload`, `headers`, and
+ * `userData` fields that will be applied to every request created from that entry.
+ *
+ * Usage:
+ *   ```javascript
+ *   import { Actor, ScrapelyRequestList } from 'scrapely';
+ *
+ *   const input = await Actor.getInput() ?? {};
+ *   const requestList = await ScrapelyRequestList.open(input.requestListSources ?? []);
+ *   // requestList.requests -> crawlee Request[]
+ *   ```
+ */
+import { Request } from '@crawlee/core';
+/**
+ * A single entry of the `requestListSources` input format.
+ * Accepts both camelCase (Apify/Scrapely UI) and snake_case keys.
+ */
+export interface RequestListSource {
+    /** Direct URL to crawl (mutually exclusive with `requestsFromUrl`). */
+    url?: string;
+    /** Remote resource whose body is scanned for URLs to crawl. */
+    requestsFromUrl?: string;
+    requests_from_url?: string;
+    /** HTTP method applied to all requests from this entry (default: GET). */
+    method?: string;
+    /** HTTP payload applied to all requests from this entry. */
+    payload?: string;
+    /** HTTP headers applied to all requests from this entry. */
+    headers?: Record<string, string>;
+    /** Custom user data applied to all requests from this entry. */
+    userData?: Record<string, any>;
+    user_data?: Record<string, any>;
+}
+export declare class ScrapelyRequestList {
+    /** Resolved requests, ready to be added to a RequestQueue or crawler. */
+    readonly requests: Request[];
+    private constructor();
+    /**
+     * Create a new `ScrapelyRequestList` from the standard Scrapely request list sources input.
+     *
+     * @param requestListSourcesInput List of entries with either a `url` key (direct URL) or a
+     * `requestsFromUrl` key (remote resource scanned for URLs). When not provided or empty,
+     * an empty request list is returned.
+     */
+    static open(requestListSourcesInput?: RequestListSource[]): Promise<ScrapelyRequestList>;
+}

package/dist/request-list.js

@@ -0,0 +1,79 @@
+"use strict";
+/**
+ * ScrapelyRequestList - parses the standard Scrapely/Apify `requestListSources` input format.
+ *
+ * Parity port of the Python SDK's `scrapely.request_loaders.ScrapelyRequestList`.
+ * It supports two kinds of entries:
+ *
+ * - **Direct URLs** - entries with a `url` key are converted to requests directly.
+ * - **Remote URL lists** - entries with a `requestsFromUrl` key point to a remote resource
+ *   (e.g. a plain-text file). The resource is fetched and all URLs found in the response
+ *   body are extracted and converted to requests.
+ *
+ * Both kinds of entries can optionally specify `method`, `payload`, `headers`, and
+ * `userData` fields that will be applied to every request created from that entry.
+ *
+ * Usage:
+ *   ```javascript
+ *   import { Actor, ScrapelyRequestList } from 'scrapely';
+ *
+ *   const input = await Actor.getInput() ?? {};
+ *   const requestList = await ScrapelyRequestList.open(input.requestListSources ?? []);
+ *   // requestList.requests -> crawlee Request[]
+ *   ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScrapelyRequestList = void 0;
+const core_1 = require("@crawlee/core");
+/** Regex for extracting URLs from remote text files (matches URLs without commas). */
+const URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([^\W_]|[^\W_][-\w0-9@:%._+~#=]{0,254}[^\W_])\.[a-z]{2,63}(:\d{1,5})?(\/[-\w@:%+.~#?&/=()]*)?/g;
+function createRequest(source, url) {
+    return new core_1.Request({
+        url,
+        method: source.method,
+        payload: source.payload,
+        headers: source.headers,
+        // Deep-copy so Request's constructor (which writes __crawlee into the
+        // dict) cannot corrupt the shared input, and nested values are not
+        // aliased across requests (Python SDK parity: deepcopy).
+        userData: structuredClone(source.userData ?? source.user_data ?? {}),
+    });
+}
+async function fetchRequestsFromUrl(source) {
+    const remoteUrl = source.requestsFromUrl ?? source.requests_from_url;
+    if (!remoteUrl)
+        return [];
+    const response = await fetch(remoteUrl);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch requestsFromUrl "${remoteUrl}": ${response.status} ${response.statusText}`);
+    }
+    const body = await response.text();
+    const requests = [];
+    for (const match of body.matchAll(URL_NO_COMMAS_REGEX)) {
+        requests.push(createRequest(source, match[0]));
+    }
+    return requests;
+}
+class ScrapelyRequestList {
+    /** Resolved requests, ready to be added to a RequestQueue or crawler. */
+    requests;
+    constructor(requests) {
+        this.requests = requests;
+    }
+    /**
+     * Create a new `ScrapelyRequestList` from the standard Scrapely request list sources input.
+     *
+     * @param requestListSourcesInput List of entries with either a `url` key (direct URL) or a
+     * `requestsFromUrl` key (remote resource scanned for URLs). When not provided or empty,
+     * an empty request list is returned.
+     */
+    static async open(requestListSourcesInput) {
+        const sources = requestListSourcesInput ?? [];
+        const simpleUrlSources = sources.filter((source) => source.url);
+        const remoteUrlSources = sources.filter((source) => !source.url && (source.requestsFromUrl ?? source.requests_from_url));
+        const simpleUrlRequests = simpleUrlSources.map((source) => createRequest(source, source.url));
+        const remoteUrlRequests = (await Promise.all(remoteUrlSources.map(fetchRequestsFromUrl))).flat();
+        return new ScrapelyRequestList([...simpleUrlRequests, ...remoteUrlRequests]);
+    }
+}
+exports.ScrapelyRequestList = ScrapelyRequestList;

package/dist/runner/docker-manager.d.ts

@@ -0,0 +1,88 @@
+import Docker from 'dockerode';
+import { TimeoutManager } from './timeout-manager.js';
+import { DragonflyClient } from '../storage/dragonfly-client.js';
+import type { UserPlan } from '../storage/ttl-manager.js';
+export interface RunOptions {
+    /** Path to the actor package directory */
+    packagePath: string;
+    /** Path to the .env file */
+    envPath?: string;
+    /** Environment variables to pass to container */
+    env?: Record<string, string>;
+    /** Maximum memory in MB (default: from manifest or 512) */
+    memoryMb?: number;
+    /** Maximum timeout in seconds (default: from manifest or 1800) */
+    timeout?: number;
+    /** Run ID for tracking */
+    runId?: string;
+    /** User ID */
+    userId?: string;
+    /** User plan for TTL */
+    userPlan?: UserPlan;
+    /** Dragonfly client for storage cleanup */
+    dragonflyClient?: DragonflyClient;
+}
+export interface RunResult {
+    containerId: string;
+    exitCode: number;
+    logs: string;
+    status: 'success' | 'failed' | 'timed_out' | 'killed';
+    duration: number;
+    /** Storage IDs that were cleaned up */
+    cleanedUp?: {
+        requestQueueDeleted: boolean;
+        logsUploadedToS3: boolean;
+    };
+}
+export declare class DockerManager {
+    private docker;
+    private timeoutManager;
+    private imageName;
+    constructor(options?: {
+        socketPath?: string;
+        imageName?: string;
+    });
+    /**
+     * Run an actor package in a container
+     */
+    run(options: RunOptions): Promise<RunResult>;
+    /**
+     * Cleanup storage after run completes.
+     * - Deletes request queue immediately (no longer needed)
+     * - Uploads logs to S3 for long-term storage
+     * - Updates run metadata with final status
+     */
+    private cleanupAfterRun;
+    /**
+     * Stop a container (SIGTERM)
+     */
+    stop(containerId: string, timeoutSeconds?: number): Promise<void>;
+    /**
+     * Kill a container (SIGKILL)
+     */
+    kill(containerId: string): Promise<void>;
+    /**
+     * Get logs from a container
+     */
+    getLogs(containerId: string): Promise<string>;
+    /**
+     * Get container status
+     */
+    getStatus(containerId: string): Promise<string | null>;
+    /**
+     * List all scrapely containers
+     */
+    listContainers(): Promise<Docker.ContainerInfo[]>;
+    /**
+     * Get the timeout manager
+     */
+    getTimeoutManager(): TimeoutManager;
+    /**
+     * Track internal data transfer from container network stats.
+     * Following Apify's model:
+     * - Ingress (rx_bytes) is FREE - scraped content coming in
+     * - Internal transfer is tracked separately in storage clients
+     * - External egress is tracked in API layer when users fetch results
+     */
+    private trackDataTransfer;
+}