scrapely 1.2.3 → 1.2.5

package/dist/storage/s3-client.d.ts

@@ -0,0 +1,103 @@
+import type { UserPlan } from './ttl-manager.js';
+/**
+ * S3 storage configuration for Hetzner Object Storage
+ */
+export interface S3Config {
+    endpoint: string;
+    accessKey: string;
+    secretKey: string;
+    bucket: string;
+    region?: string;
+}
+/**
+ * Metadata stored with each S3 log object
+ */
+export interface LogMetadata {
+    userId: string;
+    runId: string;
+    createdAt: string;
+    expiresAt: string;
+    userPlan: UserPlan;
+    contentType: string;
+    size?: number;
+}
+/**
+ * S3 client for log storage with Hetzner Object Storage support.
+ *
+ * Storage model:
+ * - Path: logs/{userId}/{runId}.log
+ * - Metadata includes TTL info for cleanup job
+ * - Supports streaming for large logs
+ */
+export declare class S3LogClient {
+    private client;
+    private bucket;
+    constructor(config: S3Config);
+    /**
+     * Get the S3 key for a run's log
+     */
+    private getLogKey;
+    /**
+     * Calculate expiration timestamp based on user plan
+     */
+    private calculateExpiresAt;
+    /**
+     * Upload log content to S3 with metadata for TTL tracking.
+     * Called when a run completes.
+     */
+    uploadLog(userId: string, runId: string, content: string | Buffer, userPlan: UserPlan): Promise<{
+        key: string;
+        metadata: LogMetadata;
+    }>;
+    /**
+     * Get log content from S3.
+     * Returns null if not found.
+     */
+    getLog(userId: string, runId: string): Promise<string | null>;
+    /**
+     * Get log as a stream for large logs.
+     */
+    getLogStream(userId: string, runId: string): Promise<NodeJS.ReadableStream | null>;
+    /**
+     * Get metadata for a log object without fetching content.
+     */
+    getLogMetadata(userId: string, runId: string): Promise<LogMetadata | null>;
+    /**
+     * Delete a log from S3.
+     */
+    deleteLog(userId: string, runId: string): Promise<boolean>;
+    /**
+     * List logs that have expired (for cleanup job).
+     * Returns up to 1000 expired logs at a time.
+     */
+    listExpiredLogs(): Promise<Array<{
+        key: string;
+        userId: string;
+        runId: string;
+        expiresAt: string;
+    }>>;
+    /**
+     * Delete multiple expired logs in batch.
+     * Returns count of deleted objects.
+     */
+    deleteExpiredLogs(): Promise<{
+        deleted: number;
+        errors: number;
+    }>;
+    /**
+     * Check if S3 is properly configured and accessible.
+     */
+    healthCheck(): Promise<boolean>;
+    /**
+     * Create S3Config from environment variables.
+     */
+    static fromEnv(): S3Config;
+}
+/**
+ * Get or create the S3 log client singleton.
+ */
+export declare function getS3LogClient(): S3LogClient | null;
+/**
+ * Initialize S3 client with explicit config.
+ */
+export declare function initS3LogClient(config: S3Config): S3LogClient;

package/dist/storage/s3-client.js

@@ -0,0 +1,302 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3LogClient = void 0;
+exports.getS3LogClient = getS3LogClient;
+exports.initS3LogClient = initS3LogClient;
+const client_s3_1 = require("@aws-sdk/client-s3");
+/**
+ * TTL for logs based on user plan (in seconds)
+ */
+const LOG_TTL = {
+    free: 7 * 24 * 60 * 60, // 7 days
+    paid: 31 * 24 * 60 * 60, // 31 days
+};
+/**
+ * S3 client for log storage with Hetzner Object Storage support.
+ *
+ * Storage model:
+ * - Path: logs/{userId}/{runId}.log
+ * - Metadata includes TTL info for cleanup job
+ * - Supports streaming for large logs
+ */
+class S3LogClient {
+    client;
+    bucket;
+    constructor(config) {
+        this.bucket = config.bucket;
+        this.client = new client_s3_1.S3Client({
+            endpoint: config.endpoint,
+            region: config.region || 'us-east-1',
+            credentials: {
+                accessKeyId: config.accessKey,
+                secretAccessKey: config.secretKey,
+            },
+            forcePathStyle: true, // Required for Hetzner S3-compatible storage
+        });
+    }
+    /**
+     * Get the S3 key for a run's log
+     */
+    getLogKey(userId, runId) {
+        return `logs/${userId}/${runId}.log`;
+    }
+    /**
+     * Calculate expiration timestamp based on user plan
+     */
+    calculateExpiresAt(userPlan) {
+        const ttlSeconds = LOG_TTL[userPlan];
+        const expiresAt = new Date(Date.now() + ttlSeconds * 1000);
+        return expiresAt.toISOString();
+    }
+    /**
+     * Upload log content to S3 with metadata for TTL tracking.
+     * Called when a run completes.
+     */
+    async uploadLog(userId, runId, content, userPlan) {
+        const now = new Date();
+        const key = this.getLogKey(userId, runId);
+        const expiresAt = this.calculateExpiresAt(userPlan);
+        const buffer = typeof content === 'string' ? Buffer.from(content, 'utf-8') : content;
+        const metadata = {
+            userId,
+            runId,
+            createdAt: now.toISOString(),
+            expiresAt,
+            userPlan,
+            contentType: 'text/plain; charset=utf-8',
+            size: buffer.length,
+        };
+        await this.client.send(new client_s3_1.PutObjectCommand({
+            Bucket: this.bucket,
+            Key: key,
+            Body: buffer,
+            ContentType: 'text/plain; charset=utf-8',
+            Metadata: {
+                'user-id': userId,
+                'run-id': runId,
+                'created-at': metadata.createdAt,
+                'expires-at': expiresAt,
+                'user-plan': userPlan,
+            },
+        }));
+        return { key, metadata };
+    }
+    /**
+     * Get log content from S3.
+     * Returns null if not found.
+     */
+    async getLog(userId, runId) {
+        const key = this.getLogKey(userId, runId);
+        try {
+            const response = await this.client.send(new client_s3_1.GetObjectCommand({
+                Bucket: this.bucket,
+                Key: key,
+            }));
+            if (!response.Body)
+                return null;
+            const content = await response.Body.transformToString('utf-8');
+            return content;
+        }
+        catch (error) {
+            if (error.name === 'NoSuchKey' || error.$metadata?.httpStatusCode === 404) {
+                return null;
+            }
+            throw error;
+        }
+    }
+    /**
+     * Get log as a stream for large logs.
+     */
+    async getLogStream(userId, runId) {
+        const key = this.getLogKey(userId, runId);
+        try {
+            const response = await this.client.send(new client_s3_1.GetObjectCommand({
+                Bucket: this.bucket,
+                Key: key,
+            }));
+            if (!response.Body)
+                return null;
+            // Convert WebStream to NodeJS ReadableStream if needed
+            return response.Body;
+        }
+        catch (error) {
+            if (error.name === 'NoSuchKey' || error.$metadata?.httpStatusCode === 404) {
+                return null;
+            }
+            throw error;
+        }
+    }
+    /**
+     * Get metadata for a log object without fetching content.
+     */
+    async getLogMetadata(userId, runId) {
+        const key = this.getLogKey(userId, runId);
+        try {
+            const response = await this.client.send(new client_s3_1.HeadObjectCommand({
+                Bucket: this.bucket,
+                Key: key,
+            }));
+            const metadata = response.Metadata || {};
+            return {
+                userId: metadata['user-id'] || userId,
+                runId: metadata['run-id'] || runId,
+                createdAt: metadata['created-at'] || new Date().toISOString(),
+                expiresAt: metadata['expires-at'] || '',
+                userPlan: metadata['user-plan'] || 'free',
+                contentType: 'text/plain; charset=utf-8',
+                size: response.ContentLength,
+            };
+        }
+        catch (error) {
+            if (error.$metadata?.httpStatusCode === 404) {
+                return null;
+            }
+            throw error;
+        }
+    }
+    /**
+     * Delete a log from S3.
+     */
+    async deleteLog(userId, runId) {
+        const key = this.getLogKey(userId, runId);
+        try {
+            await this.client.send(new client_s3_1.DeleteObjectCommand({
+                Bucket: this.bucket,
+                Key: key,
+            }));
+            return true;
+        }
+        catch (error) {
+            if (error.$metadata?.httpStatusCode === 404) {
+                return false;
+            }
+            throw error;
+        }
+    }
+    /**
+     * List logs that have expired (for cleanup job).
+     * Returns up to 1000 expired logs at a time.
+     */
+    async listExpiredLogs() {
+        const now = new Date();
+        const expiredLogs = [];
+        let continuationToken;
+        do {
+            const response = await this.client.send(new client_s3_1.ListObjectsV2Command({
+                Bucket: this.bucket,
+                Prefix: 'logs/',
+                ContinuationToken: continuationToken,
+            }));
+            if (!response.Contents)
+                break;
+            for (const object of response.Contents) {
+                if (!object.Key)
+                    continue;
+                // Parse metadata to check expiration
+                try {
+                    const headResponse = await this.client.send(new client_s3_1.HeadObjectCommand({
+                        Bucket: this.bucket,
+                        Key: object.Key,
+                    }));
+                    const expiresAtStr = headResponse.Metadata?.['expires-at'];
+                    if (!expiresAtStr)
+                        continue;
+                    const expiresAt = new Date(expiresAtStr);
+                    if (expiresAt <= now) {
+                        // Parse userId and runId from key: logs/{userId}/{runId}.log
+                        const parts = object.Key.split('/');
+                        if (parts.length >= 3) {
+                            const userId = parts[1];
+                            const runId = parts[2].replace('.log', '');
+                            expiredLogs.push({
+                                key: object.Key,
+                                userId,
+                                runId,
+                                expiresAt: expiresAtStr,
+                            });
+                        }
+                    }
+                }
+                catch {
+                    // Skip objects we can't read metadata for
+                    continue;
+                }
+            }
+            continuationToken = response.NextContinuationToken;
+        } while (continuationToken);
+        return expiredLogs;
+    }
+    /**
+     * Delete multiple expired logs in batch.
+     * Returns count of deleted objects.
+     */
+    async deleteExpiredLogs() {
+        const expiredLogs = await this.listExpiredLogs();
+        let deleted = 0;
+        let errors = 0;
+        for (const log of expiredLogs) {
+            try {
+                await this.deleteLog(log.userId, log.runId);
+                deleted++;
+            }
+            catch {
+                errors++;
+            }
+        }
+        return { deleted, errors };
+    }
+    /**
+     * Check if S3 is properly configured and accessible.
+     */
+    async healthCheck() {
+        try {
+            await this.client.send(new client_s3_1.HeadObjectCommand({
+                Bucket: this.bucket,
+                Key: '__health_check__',
+            }));
+            return true; // Object exists or bucket is accessible
+        }
+        catch (error) {
+            // NoSuchKey means bucket exists but object doesn't - still healthy
+            if (error.name === 'NoSuchKey' || error.$metadata?.httpStatusCode === 404) {
+                return true;
+            }
+            return false;
+        }
+    }
+    /**
+     * Create S3Config from environment variables.
+     */
+    static fromEnv() {
+        return {
+            endpoint: process.env.S3_ENDPOINT || 'https://nbg1.your-objectstorage.com',
+            accessKey: process.env.S3_ACCESS_KEY || '',
+            secretKey: process.env.S3_SECRET_KEY || '',
+            bucket: process.env.S3_BUCKET || 'scrapely-logs',
+            region: process.env.S3_REGION || 'us-east-1',
+        };
+    }
+}
+exports.S3LogClient = S3LogClient;
+// Singleton instance
+let s3LogClient = null;
+/**
+ * Get or create the S3 log client singleton.
+ */
+function getS3LogClient() {
+    const config = S3LogClient.fromEnv();
+    if (!config.accessKey || !config.secretKey) {
+        return null;
+    }
+    if (!s3LogClient) {
+        s3LogClient = new S3LogClient(config);
+    }
+    return s3LogClient;
+}
+/**
+ * Initialize S3 client with explicit config.
+ */
+function initS3LogClient(config) {
+    s3LogClient = new S3LogClient(config);
+    return s3LogClient;
+}

package/dist/storage/ttl-manager.d.ts

@@ -0,0 +1,70 @@
+import type Redis from 'ioredis';
+/**
+ * TTL configuration in seconds
+ */
+export interface TtlConfig {
+    /** TTL for unnamed datasets (free users) - default 7 days */
+    datasetFree: number;
+    /** TTL for unnamed datasets (paid users) - default 31 days */
+    datasetPaid: number;
+    /** TTL for unnamed KV stores (free users) - default 7 days */
+    kvFree: number;
+    /** TTL for unnamed KV stores (paid users) - default 31 days */
+    kvPaid: number;
+    /** Fallback TTL for request queues (safety net for orphaned RQs) - default 24 hours */
+    requestQueueFallback: number;
+    /** TTL for logs in Redis (free users) - default 12 hours */
+    logsFree: number;
+    /** TTL for logs in Redis (paid users) - default 7 days */
+    logsPaid: number;
+    /** Maximum log size in bytes - default 5MB */
+    logMaxSize: number;
+}
+/**
+ * Default TTL configuration based on Apify's model
+ */
+export declare const DEFAULT_TTL_CONFIG: TtlConfig;
+export type StorageType = 'dataset' | 'kv' | 'rq' | 'logs';
+export type UserPlan = 'free' | 'paid';
+/**
+ * Manages TTL (Time-To-Live) for Dragonfly storage keys.
+ *
+ * Retention policy:
+ * - Named storage: Indefinite (billed), no TTL set
+ * - Unnamed datasets/KV stores: 7 days (free) or 31 days (paid)
+ * - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
+ * - Logs: Short TTL in Redis, persisted to S3
+ */
+export declare class TtlManager {
+    private config;
+    constructor(config?: Partial<TtlConfig>);
+    /**
+     * Get the TTL for a storage type based on user plan and whether it's named.
+     * Returns null for named storage (no TTL = indefinite).
+     */
+    getTtl(storageType: 'dataset' | 'kv', plan: UserPlan, isNamed: boolean): number | null;
+    getTtl(storageType: 'rq', plan: UserPlan): number;
+    getTtl(storageType: 'logs', plan: UserPlan): number;
+    /**
+     * Set expiry on multiple keys atomically using a pipeline.
+     * Uses 'NX' flag to only set TTL if it doesn't already exist.
+     */
+    setExpiry(redis: Redis, keys: string[], ttl: number): Promise<void>;
+    /**
+     * Set expiry on multiple keys, overwriting any existing TTL.
+     * Use this when you want to guarantee the TTL is set to a specific value.
+     */
+    setExpiryForce(redis: Redis, keys: string[], ttl: number): Promise<void>;
+    /**
+     * Get the maximum log size in bytes
+     */
+    getLogMaxSize(): number;
+    /**
+     * Get the full TTL config
+     */
+    getConfig(): Readonly<TtlConfig>;
+    /**
+     * Create TtlConfig from environment variables
+     */
+    static fromEnv(): TtlConfig;
+}

package/dist/storage/ttl-manager.js

@@ -0,0 +1,115 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TtlManager = exports.DEFAULT_TTL_CONFIG = void 0;
+/**
+ * Default TTL configuration based on Apify's model
+ */
+exports.DEFAULT_TTL_CONFIG = {
+    datasetFree: 7 * 24 * 60 * 60, // 7 days
+    datasetPaid: 31 * 24 * 60 * 60, // 31 days
+    kvFree: 7 * 24 * 60 * 60, // 7 days
+    kvPaid: 31 * 24 * 60 * 60, // 31 days
+    requestQueueFallback: 24 * 60 * 60, // 24 hours
+    logsFree: 12 * 60 * 60, // 12 hours
+    logsPaid: 7 * 24 * 60 * 60, // 7 days
+    logMaxSize: 5 * 1024 * 1024, // 5MB
+};
+/**
+ * Manages TTL (Time-To-Live) for Dragonfly storage keys.
+ *
+ * Retention policy:
+ * - Named storage: Indefinite (billed), no TTL set
+ * - Unnamed datasets/KV stores: 7 days (free) or 31 days (paid)
+ * - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
+ * - Logs: Short TTL in Redis, persisted to S3
+ */
+class TtlManager {
+    config;
+    constructor(config) {
+        this.config = { ...exports.DEFAULT_TTL_CONFIG, ...config };
+    }
+    getTtl(storageType, plan, isNamed) {
+        // Named storage gets no TTL (indefinite retention, billed)
+        if ((storageType === 'dataset' || storageType === 'kv') && isNamed) {
+            return null;
+        }
+        switch (storageType) {
+            case 'dataset':
+                return plan === 'paid' ? this.config.datasetPaid : this.config.datasetFree;
+            case 'kv':
+                return plan === 'paid' ? this.config.kvPaid : this.config.kvFree;
+            case 'rq':
+                return this.config.requestQueueFallback;
+            case 'logs':
+                return plan === 'paid' ? this.config.logsPaid : this.config.logsFree;
+            default:
+                return null;
+        }
+    }
+    /**
+     * Set expiry on multiple keys atomically using a pipeline.
+     * Uses 'NX' flag to only set TTL if it doesn't already exist.
+     */
+    async setExpiry(redis, keys, ttl) {
+        if (keys.length === 0 || ttl <= 0)
+            return;
+        const pipeline = redis.pipeline();
+        for (const key of keys) {
+            // 'NX' = only set expiry if key has no expiry
+            // This prevents accidentally shortening TTL on access
+            pipeline.expire(key, ttl, 'NX');
+        }
+        await pipeline.exec();
+    }
+    /**
+     * Set expiry on multiple keys, overwriting any existing TTL.
+     * Use this when you want to guarantee the TTL is set to a specific value.
+     */
+    async setExpiryForce(redis, keys, ttl) {
+        if (keys.length === 0 || ttl <= 0)
+            return;
+        const pipeline = redis.pipeline();
+        for (const key of keys) {
+            pipeline.expire(key, ttl);
+        }
+        await pipeline.exec();
+    }
+    /**
+     * Get the maximum log size in bytes
+     */
+    getLogMaxSize() {
+        return this.config.logMaxSize;
+    }
+    /**
+     * Get the full TTL config
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+    /**
+     * Create TtlConfig from environment variables
+     */
+    static fromEnv() {
+        const parseEnv = (key, defaultValue) => {
+            const val = process.env[key];
+            if (val) {
+                const parsed = parseInt(val, 10);
+                if (!isNaN(parsed) && parsed > 0)
+                    return parsed;
+            }
+            return defaultValue;
+        };
+        return {
+            datasetFree: parseEnv('SCRAPELY_TTL_DATASET_FREE', exports.DEFAULT_TTL_CONFIG.datasetFree),
+            datasetPaid: parseEnv('SCRAPELY_TTL_DATASET_PAID', exports.DEFAULT_TTL_CONFIG.datasetPaid),
+            // Bug fix: these two env vars were silently ignored before
+            kvFree: parseEnv('SCRAPELY_TTL_KV_FREE', exports.DEFAULT_TTL_CONFIG.kvFree),
+            kvPaid: parseEnv('SCRAPELY_TTL_KV_PAID', exports.DEFAULT_TTL_CONFIG.kvPaid),
+            requestQueueFallback: parseEnv('SCRAPELY_TTL_RQ_FALLBACK', exports.DEFAULT_TTL_CONFIG.requestQueueFallback),
+            logsFree: parseEnv('SCRAPELY_TTL_LOGS_FREE', exports.DEFAULT_TTL_CONFIG.logsFree),
+            logsPaid: parseEnv('SCRAPELY_TTL_LOGS_PAID', exports.DEFAULT_TTL_CONFIG.logsPaid),
+            logMaxSize: parseEnv('SCRAPELY_LOG_MAX_SIZE', exports.DEFAULT_TTL_CONFIG.logMaxSize),
+        };
+    }
+}
+exports.TtlManager = TtlManager;

package/dist/storage/utils.d.ts

@@ -0,0 +1,3 @@
+export declare function uniqueKeyToRequestId(uniqueKey: string): string;
+export declare function generateId(): string;
+export declare function nowISO(): Date;

package/dist/storage/utils.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uniqueKeyToRequestId = uniqueKeyToRequestId;
+exports.generateId = generateId;
+exports.nowISO = nowISO;
+const node_crypto_1 = require("node:crypto");
+function uniqueKeyToRequestId(uniqueKey) {
+    return (0, node_crypto_1.createHash)('sha256')
+        .update(uniqueKey)
+        .digest('base64')
+        .replace(/(\+|\/|=)/g, '')
+        .slice(0, 15);
+}
+function generateId() {
+    return (0, node_crypto_1.randomUUID)();
+}
+function nowISO() {
+    return new Date();
+}

package/dist/storage-identifiers.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Storage identifier types (Apify-compatible names, so TypeScript actors importing
+ * them from 'apify' still compile after the apify → scrapely rewrite).
+ */
+/** Identifies a storage by its alias from the Actor's schema storages. */
+export interface StorageAlias {
+    alias: string;
+}
+/** Identifies a storage by its platform ID. */
+export interface StorageId {
+    id: string;
+}
+/** Identifies a storage by its name. */
+export interface StorageName {
+    name: string;
+}
+/** A plain string (ID or name), `{ alias }`, `{ id }` or `{ name }`. */
+export type StorageIdentifier = string | StorageAlias | StorageId | StorageName;
+/** A plain string (ID or name), `{ id }` or `{ name }` (key-value stores and request queues). */
+export type StorageIdentifierWithoutAlias = string | StorageId | StorageName;

package/dist/storage-identifiers.js

@@ -0,0 +1,6 @@
+"use strict";
+/**
+ * Storage identifier types (Apify-compatible names, so TypeScript actors importing
+ * them from 'apify' still compile after the apify → scrapely rewrite).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Apify v2-compatible utility functions for scrapely.
+ * These adapt the Apify v2 API to Crawlee v3 internally.
+ */
+import type { CheerioAPI } from 'cheerio';
+import type { RequestQueue } from 'crawlee';
+/**
+ * Options for enqueueLinks (Apify v2 API compatible).
+ */
+export interface EnqueueLinksOptions {
+    /** Cheerio instance ($) - REQUIRED in Apify v2 */
+    $: CheerioAPI;
+    /** RequestQueue instance - REQUIRED in Apify v2 */
+    requestQueue: RequestQueue;
+    /** CSS selector to find links. Default: 'a[href]' */
+    selector?: string;
+    /** Base URL for resolving relative URLs */
+    baseUrl?: string;
+    /** Optional userData to merge into the queued requests */
+    userData?: Record<string, any>;
+    /** Optional transformation function for each URL */
+    transformRequestFunction?: (request: {
+        url: string;
+        userData?: Record<string, any>;
+    }) => {
+        url: string;
+        userData?: Record<string, any>;
+    } | null;
+}
+/**
+ * Enqueues links found on the page using a CSS selector.
+ *
+ * This is an Apify v2-compatible implementation that adapts to Crawlee v3 internally.
+ * It extracts URLs from the page using cheerio and adds them to the request queue.
+ *
+ * @example
+ * ```typescript
+ * // Inside a CheerioCrawler requestHandler:
+ * await Apify.utils.enqueueLinks({
+ *     $,
+ *     requestQueue,
+ *     selector: 'a[href*="/app/"]',
+ *     baseUrl: new URL(request.url).origin,
+ * });
+ * ```
+ */
+export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<void>;
+/**
+ * Utility namespace providing Apify v2-compatible helper functions.
+ * These adapt to Crawlee v3 APIs internally.
+ */
+export declare const utils: {
+    enqueueLinks: typeof enqueueLinks;
+};