scrapely 1.2.3 → 1.2.4

package/dist/storage/dataset-client.js

@@ -0,0 +1,232 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DragonflyDatasetClient = void 0;
+const utils_js_1 = require("./utils.js");
+const billing_tracker_js_1 = require("./billing-tracker.js");
+const dataset_push_script_js_1 = require("./dataset-push-script.js");
+function makeAsyncIterableResult(promiseFn) {
+    const promise = promiseFn();
+    const iterable = {
+        [Symbol.asyncIterator]: () => {
+            let items = null;
+            let index = 0;
+            return {
+                async next() {
+                    if (!items)
+                        items = (await promise).items;
+                    if (index >= items.length)
+                        return { done: true, value: undefined };
+                    return { done: false, value: items[index++] };
+                },
+            };
+        },
+        then: promise.then.bind(promise),
+        catch: promise.catch.bind(promise),
+        finally: promise.finally.bind(promise),
+        [Symbol.toStringTag]: 'AsyncIterablePromise',
+    };
+    return iterable;
+}
+class DragonflyDatasetClient {
+    client;
+    id;
+    infoKey;
+    itemsKey;
+    constructor(client, id) {
+        this.client = client;
+        this.id = id;
+        this.infoKey = client.key('dataset', id, 'info');
+        this.itemsKey = client.key('dataset', id, 'items');
+    }
+    async get() {
+        const info = await this.client.redis.hgetall(this.infoKey);
+        if (!info || !info.id)
+            return undefined;
+        return {
+            id: info.id,
+            name: info.name || undefined,
+            createdAt: new Date(info.createdAt),
+            modifiedAt: new Date(info.modifiedAt),
+            accessedAt: new Date(info.accessedAt),
+            itemCount: parseInt(info.itemCount || '0', 10),
+            storageBytes: parseInt(info.storageBytes || '0', 10),
+        };
+    }
+    async update(newFields) {
+        const now = (0, utils_js_1.nowISO)();
+        if (newFields.name !== undefined) {
+            await this.client.redis.hset(this.infoKey, 'name', newFields.name);
+            const indexKey = this.client.key('datasets:index');
+            const raw = await this.client.redis.hget(indexKey, `id:${this.id}`);
+            if (raw) {
+                const data = JSON.parse(raw);
+                if (data.name)
+                    await this.client.redis.hdel(indexKey, `name:${data.name}`);
+                data.name = newFields.name;
+                await this.client.redis.hset(indexKey, `id:${this.id}`, JSON.stringify(data));
+                await this.client.redis.hset(indexKey, `name:${newFields.name}`, JSON.stringify(data));
+            }
+        }
+        await this.client.redis.hset(this.infoKey, 'modifiedAt', now.toISOString());
+        return { name: newFields.name };
+    }
+    async delete() {
+        const info = await this.client.redis.hgetall(this.infoKey);
+        await this.client.redis.del(this.infoKey, this.itemsKey);
+        const indexKey = this.client.key('datasets:index');
+        await this.client.redis.hdel(indexKey, `id:${this.id}`);
+        if (info?.name)
+            await this.client.redis.hdel(indexKey, `name:${info.name}`);
+    }
+    async downloadItems() {
+        const result = await this._listItems();
+        return Buffer.from(JSON.stringify(result.items));
+    }
+    listItems(options) {
+        return makeAsyncIterableResult(() => this._listItems(options));
+    }
+    /**
+     * Iterates over dataset entries (index-value pairs) using an async generator.
+     * Required by Crawlee's Dataset.entries() method.
+     */
+    listEntries(options) {
+        return makeAsyncIterableResult(async () => {
+            const result = await this._listItems(options);
+            const entries = result.items.map((item, idx) => [
+                (result.offset ?? 0) + idx,
+                item,
+            ]);
+            return {
+                total: result.total,
+                count: entries.length,
+                offset: result.offset,
+                limit: result.limit,
+                items: entries,
+            };
+        });
+    }
+    async _listItems(options) {
+        const exists = await this.client.redis.exists(this.infoKey);
+        if (!exists)
+            return { total: 0, count: 0, offset: 0, limit: 0, items: [] };
+        // Item indices are dense (0..N-1: pushes append sequentially and there is
+        // no per-item delete), so pages can be addressed by index directly instead
+        // of pulling the whole hash on every request (matches the server route).
+        const totalCount = await this.client.redis.hlen(this.itemsKey);
+        const desc = options?.desc ?? false;
+        const offset = options?.offset ?? 0;
+        const limit = options?.limit ?? totalCount;
+        const pageIndices = [];
+        for (let n = 0; n < limit && offset + n < totalCount; n++) {
+            pageIndices.push(desc ? totalCount - 1 - offset - n : offset + n);
+        }
+        let items = pageIndices.length
+            ? (await this.client.redis.hmget(this.itemsKey, ...pageIndices.map(String)))
+                .filter((raw) => raw !== null)
+                .map(raw => JSON.parse(raw))
+            : [];
+        // Apply filtering options
+        items = this._applyFilters(items, options);
+        // Track reads for billing - increment both user-level and info hash counters
+        if (items.length > 0 && this.client.billingContext) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            await billingTracker.trackRead(this.client.billingContext.userId, items.length);
+            await this.client.redis.hincrby(this.infoKey, 'readCount', items.length);
+            // Track internal data transfer (bytes returned to actor)
+            const bytesTransferred = items.reduce((sum, item) => sum + billing_tracker_js_1.BillingTracker.estimateItemBytes(item), 0);
+            await billingTracker.trackInternalTransfer(this.client.billingContext.runId, this.client.billingContext.userId, bytesTransferred);
+            // Register this dataset as accessed for per-run aggregation
+            await this.client.redis.sadd(this.client.key('run', this.client.billingContext.runId, 'datasets_accessed'), this.id);
+        }
+        return { total: totalCount, count: items.length, offset, limit, desc: options?.desc, items };
+    }
+    /**
+     * Apply filtering options to dataset items.
+     */
+    _applyFilters(items, options) {
+        if (!options)
+            return items;
+        return items.map(item => {
+            let filteredItem = { ...item };
+            // Skip hidden fields (fields starting with "#")
+            if (options.skipHidden || options.clean) {
+                for (const key of Object.keys(filteredItem)) {
+                    if (key.startsWith('#')) {
+                        delete filteredItem[key];
+                    }
+                }
+            }
+            // Filter to only specified fields
+            if (options.fields && options.fields.length > 0) {
+                const filtered = {};
+                for (const field of options.fields) {
+                    if (field in filteredItem) {
+                        filtered[field] = filteredItem[field];
+                    }
+                }
+                filteredItem = filtered;
+            }
+            return filteredItem;
+        }).filter(item => {
+            // Skip empty items
+            if (options.skipEmpty || options.clean) {
+                return Object.keys(item).length > 0;
+            }
+            return true;
+        });
+    }
+    async pushItems(items, billingContext) {
+        const exists = await this.client.redis.exists(this.infoKey);
+        if (!exists)
+            throw new Error(`Dataset with id: ${this.id} does not exist.`);
+        let parsed;
+        if (typeof items === 'string') {
+            try {
+                parsed = JSON.parse(items);
+            }
+            catch {
+                throw new Error('Data items must be a JSON object, array, or JSON string.');
+            }
+            if (!Array.isArray(parsed))
+                parsed = [parsed];
+        }
+        else if (Array.isArray(items)) {
+            parsed = items.map(i => (typeof i === 'string' ? JSON.parse(i) : i));
+        }
+        else if (typeof items === 'object' && items !== null) {
+            parsed = [items];
+        }
+        else {
+            throw new Error('Data items must be a JSON object, array, or JSON string.');
+        }
+        for (const item of parsed) {
+            if (typeof item !== 'object' || item === null || Array.isArray(item)) {
+                throw new Error('Each item must be a JSON object.');
+            }
+        }
+        // Calculate total bytes for billing
+        let totalBytes = 0;
+        const serializedItems = [];
+        for (let i = 0; i < parsed.length; i++) {
+            const serialized = JSON.stringify(parsed[i]);
+            serializedItems.push(serialized);
+            totalBytes += billing_tracker_js_1.BillingTracker.estimateItemBytes(parsed[i]);
+        }
+        // Atomic slot allocation: the previous read-then-write of itemCount in
+        // separate round trips allowed concurrent pushes to allocate the same
+        // slot indices, silently overwriting items.
+        await (0, dataset_push_script_js_1.pushItemsAtomic)(this.client.redis, this.infoKey, this.itemsKey, serializedItems, totalBytes);
+        // Track billing using client's billing context
+        if (this.client.billingContext) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            await billingTracker.trackStorageBytes(this.client.billingContext.runId, this.client.billingContext.userId, totalBytes);
+            await billingTracker.trackWrite(this.client.billingContext.userId, parsed.length);
+            await this.client.redis.hincrby(this.infoKey, 'writeCount', parsed.length);
+            // Track internal data transfer (bytes written to storage)
+            await billingTracker.trackInternalTransfer(this.client.billingContext.runId, this.client.billingContext.userId, totalBytes);
+            // Register this dataset as accessed for per-run aggregation
+            await this.client.redis.sadd(this.client.key('run', this.client.billingContext.runId, 'datasets_accessed'), this.id);
+        }
+    }
+}
+exports.DragonflyDatasetClient = DragonflyDatasetClient;

package/dist/storage/dataset-collection-client.d.ts

@@ -0,0 +1,8 @@
+import type { DatasetCollectionClient, DatasetCollectionData, PaginatedList, Dataset } from '@crawlee/types';
+import type { DragonflyClient } from './dragonfly-client.js';
+export declare class DragonflyDatasetCollectionClient implements DatasetCollectionClient {
+    private client;
+    constructor(client: DragonflyClient);
+    getOrCreate(name?: string): Promise<DatasetCollectionData>;
+    list(): Promise<PaginatedList<Dataset>>;
+}

package/dist/storage/dataset-collection-client.js

@@ -0,0 +1,75 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DragonflyDatasetCollectionClient = void 0;
+const utils_js_1 = require("./utils.js");
+class DragonflyDatasetCollectionClient {
+    client;
+    constructor(client) {
+        this.client = client;
+    }
+    async getOrCreate(name) {
+        const indexKey = this.client.key('datasets:index');
+        if (name) {
+            const existing = await this.client.redis.hget(indexKey, `name:${name}`);
+            if (existing)
+                return JSON.parse(existing);
+        }
+        const id = (0, utils_js_1.generateId)();
+        const now = (0, utils_js_1.nowISO)();
+        const data = {
+            id,
+            name,
+            createdAt: now,
+            modifiedAt: now,
+            accessedAt: now,
+        };
+        await this.client.redis.hset(indexKey, `id:${id}`, JSON.stringify(data));
+        if (name)
+            await this.client.redis.hset(indexKey, `name:${name}`, JSON.stringify(data));
+        const infoKey = this.client.key('dataset', id, 'info');
+        const itemsKey = this.client.key('dataset', id, 'items');
+        await this.client.redis.hset(infoKey, {
+            id,
+            name: name ?? '',
+            itemCount: '0',
+            createdAt: now.toISOString(),
+            modifiedAt: now.toISOString(),
+            accessedAt: now.toISOString(),
+        });
+        // Set TTL for unnamed datasets (named datasets have indefinite retention)
+        if (!name) {
+            const ttl = this.client.ttlManager.getTtl('dataset', this.client.userPlan, false);
+            if (ttl) {
+                await this.client.ttlManager.setExpiry(this.client.redis, [infoKey, itemsKey], ttl);
+            }
+        }
+        return data;
+    }
+    async list() {
+        const indexKey = this.client.key('datasets:index');
+        const all = await this.client.redis.hgetall(indexKey);
+        const items = [];
+        const seen = new Set();
+        for (const [key, val] of Object.entries(all)) {
+            if (!key.startsWith('id:'))
+                continue;
+            const parsed = JSON.parse(val);
+            if (seen.has(parsed.id))
+                continue;
+            seen.add(parsed.id);
+            const infoKey = this.client.key('dataset', parsed.id, 'info');
+            // Skip and clean up stale index entries whose keys have expired via TTL
+            const exists = await this.client.redis.exists(infoKey);
+            if (!exists) {
+                await this.client.redis.hdel(indexKey, `id:${parsed.id}`);
+                if (parsed.name)
+                    await this.client.redis.hdel(indexKey, `name:${parsed.name}`);
+                continue;
+            }
+            const itemCount = parseInt(await this.client.redis.hget(infoKey, 'itemCount') || '0', 10);
+            items.push({ ...parsed, itemCount });
+        }
+        return { total: items.length, count: items.length, offset: 0, limit: items.length, items };
+    }
+}
+exports.DragonflyDatasetCollectionClient = DragonflyDatasetCollectionClient;

package/dist/storage/dataset-push-script.d.ts

@@ -0,0 +1,18 @@
+import type Redis from 'ioredis';
+/**
+ * Atomic dataset push: allocates consecutive item slots starting at the current
+ * itemCount, writes the items, and updates itemCount/storageBytes/modifiedAt in
+ * a single script.
+ *
+ * Without this, pushItems did a non-atomic `HGET itemCount` followed by HSETs of
+ * `count..count+n` in a pipeline, so two concurrent pushes could read the same
+ * count and write overlapping slots — silently overwriting items and skewing
+ * itemCount away from HLEN (which also broke the dense-index pagination
+ * assumption used by both this client and the server route).
+ */
+export declare const PUSH_ITEMS_LUA = "\nlocal infoKey = KEYS[1]\nlocal itemsKey = KEYS[2]\nlocal now = ARGV[1]\nlocal bytes = tonumber(ARGV[2])\nlocal count = tonumber(redis.call('HGET', infoKey, 'itemCount') or '0')\nfor i = 3, #ARGV do\n  redis.call('HSET', itemsKey, tostring(count), ARGV[i])\n  count = count + 1\nend\nredis.call('HSET', infoKey, 'itemCount', tostring(count))\nredis.call('HINCRBY', infoKey, 'storageBytes', bytes)\nredis.call('HSET', infoKey, 'modifiedAt', now)\nreturn count\n";
+/**
+ * Pushes serialized items into the dataset hash, allocating dense consecutive
+ * slots atomically. Returns the new itemCount.
+ */
+export declare function pushItemsAtomic(redis: Redis, infoKey: string, itemsKey: string, serializedItems: string[], totalBytes: number): Promise<number>;

package/dist/storage/dataset-push-script.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PUSH_ITEMS_LUA = void 0;
+exports.pushItemsAtomic = pushItemsAtomic;
+/**
+ * Atomic dataset push: allocates consecutive item slots starting at the current
+ * itemCount, writes the items, and updates itemCount/storageBytes/modifiedAt in
+ * a single script.
+ *
+ * Without this, pushItems did a non-atomic `HGET itemCount` followed by HSETs of
+ * `count..count+n` in a pipeline, so two concurrent pushes could read the same
+ * count and write overlapping slots — silently overwriting items and skewing
+ * itemCount away from HLEN (which also broke the dense-index pagination
+ * assumption used by both this client and the server route).
+ */
+exports.PUSH_ITEMS_LUA = `
+local infoKey = KEYS[1]
+local itemsKey = KEYS[2]
+local now = ARGV[1]
+local bytes = tonumber(ARGV[2])
+local count = tonumber(redis.call('HGET', infoKey, 'itemCount') or '0')
+for i = 3, #ARGV do
+  redis.call('HSET', itemsKey, tostring(count), ARGV[i])
+  count = count + 1
+end
+redis.call('HSET', infoKey, 'itemCount', tostring(count))
+redis.call('HINCRBY', infoKey, 'storageBytes', bytes)
+redis.call('HSET', infoKey, 'modifiedAt', now)
+return count
+`;
+/**
+ * Pushes serialized items into the dataset hash, allocating dense consecutive
+ * slots atomically. Returns the new itemCount.
+ */
+async function pushItemsAtomic(redis, infoKey, itemsKey, serializedItems, totalBytes) {
+    const result = await redis.eval(exports.PUSH_ITEMS_LUA, 2, infoKey, itemsKey, new Date().toISOString(), String(totalBytes), ...serializedItems);
+    return typeof result === 'number' ? result : parseInt(String(result), 10);
+}

package/dist/storage/dragonfly-client.d.ts

@@ -0,0 +1,94 @@
+import Redis from 'ioredis';
+import type { StorageClient, SetStatusMessageOptions, RequestQueueOptions } from '@crawlee/types';
+import { DragonflyDatasetCollectionClient } from './dataset-collection-client.js';
+import { DragonflyDatasetClient } from './dataset-client.js';
+import { DragonflyKeyValueStoreCollectionClient } from './kv-store-collection-client.js';
+import { DragonflyKeyValueStoreClient } from './key-value-store-client.js';
+import { DragonflyRequestQueueCollectionClient } from './rq-collection-client.js';
+import { DragonflyRequestQueueClient } from './request-queue-client.js';
+import { TtlManager, TtlConfig, UserPlan } from './ttl-manager.js';
+export interface BillingContext {
+    runId: string;
+    userId: string;
+}
+export interface DragonflyClientOptions {
+    host: string;
+    port?: number;
+    password?: string;
+    keyPrefix?: string;
+    /**
+     * Single-tenant escape hatch: allow purge() to delete the entire default
+     * `crawlee:` key prefix even when no billingContext.runId is set. The
+     * `crawlee:` namespace is shared by every user/run on the platform, so
+     * purging it wholesale would delete other tenants' data — hence opt-in.
+     */
+    allowUnscopedPurge?: boolean;
+    /** User plan for TTL determination */
+    userPlan?: UserPlan;
+    /** Custom TTL configuration */
+    ttlConfig?: Partial<TtlConfig>;
+    /** Billing context for tracking usage */
+    billingContext?: BillingContext;
+}
+export declare class DragonflyClient implements StorageClient {
+    readonly redis: Redis;
+    readonly keyPrefix: string;
+    readonly ttlManager: TtlManager;
+    readonly userPlan: UserPlan;
+    readonly billingContext?: BillingContext;
+    readonly allowUnscopedPurge: boolean;
+    stats: {
+        rateLimitErrors: number[];
+    };
+    constructor(options: DragonflyClientOptions);
+    key(...parts: string[]): string;
+    datasets(): DragonflyDatasetCollectionClient;
+    dataset(id: string): DragonflyDatasetClient;
+    keyValueStores(): DragonflyKeyValueStoreCollectionClient;
+    keyValueStore(id: string): DragonflyKeyValueStoreClient;
+    requestQueues(): DragonflyRequestQueueCollectionClient;
+    requestQueue(id: string, options?: RequestQueueOptions): DragonflyRequestQueueClient;
+    /**
+     * Purges storage, mirroring Crawlee's StorageClient.purge() contract: clear
+     * only the current run's default storages (dataset, KV store, request queue).
+     *
+     * Scoping rules — the default `crawlee:` prefix is shared by every user/run
+     * on the platform (datasets, request queues, billing, auth tokens all live
+     * under it), so purging the whole prefix would delete other tenants' data:
+     *
+     *  1. With `billingContext.runId`: resolve the run's storageIds from
+     *     `{prefix}run:{runId}` and delete only those three storage namespaces.
+     *  2. Without a run ID, but with a custom (scoped) `keyPrefix`: delete the
+     *     whole prefix — it is assumed dedicated to this client.
+     *  3. Without a run ID and with the shared default prefix: refuse, unless
+     *     `allowUnscopedPurge` was set explicitly (single-tenant opt-in).
+     *
+     * Deletion uses SCAN + batched UNLINK (see deleteByPattern) instead of a
+     * blocking KEYS walk followed by a single huge DEL.
+     */
+    purge(): Promise<void>;
+    /**
+     * Delete the default storages (dataset / KV store / request queue) of a run,
+     * resolved from the run's metadata hash (`{prefix}run:{runId}` → storageIds).
+     *
+     * @returns the number of keys deleted, or null when run metadata is missing
+     *          (expired or never created) and nothing could be scoped.
+     */
+    private purgeRunDefaultStorages;
+    /**
+     * Incrementally delete every key matching `pattern` using SCAN + batched
+     * UNLINK. Unlike KEYS + DEL, this never walks the keyspace in one blocking
+     * pass, never materializes the full key list in a single reply, and never
+     * sends one gigantic delete command; UNLINK also reclaims memory
+     * asynchronously. Keys that vanish mid-scan are simply no-ops.
+     *
+     * @returns number of keys deleted (best effort).
+     */
+    deleteByPattern(pattern: string, options?: {
+        scanCount?: number;
+        unlinkBatch?: number;
+    }): Promise<number>;
+    teardown(): Promise<void>;
+    disconnect(): Promise<void>;
+    setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;
+}

package/dist/storage/dragonfly-client.js

@@ -0,0 +1,167 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DragonflyClient = void 0;
+const ioredis_1 = __importDefault(require("ioredis"));
+const dataset_collection_client_js_1 = require("./dataset-collection-client.js");
+const dataset_client_js_1 = require("./dataset-client.js");
+const kv_store_collection_client_js_1 = require("./kv-store-collection-client.js");
+const key_value_store_client_js_1 = require("./key-value-store-client.js");
+const rq_collection_client_js_1 = require("./rq-collection-client.js");
+const request_queue_client_js_1 = require("./request-queue-client.js");
+const ttl_manager_js_1 = require("./ttl-manager.js");
+/** The platform-wide storage namespace shared by all users and runs. */
+const PLATFORM_KEY_PREFIX = 'crawlee:';
+/** SCAN COUNT hint for deleteByPattern(): keys examined per scan step. */
+const DEFAULT_SCAN_COUNT = 500;
+/** Max keys per UNLINK call in deleteByPattern(): keeps command buffers small. */
+const DEFAULT_UNLINK_BATCH = 1000;
+class DragonflyClient {
+    redis;
+    keyPrefix;
+    ttlManager;
+    userPlan;
+    billingContext;
+    allowUnscopedPurge;
+    stats = { rateLimitErrors: [] };
+    constructor(options) {
+        const redisOpts = {
+            host: options.host,
+            port: options.port ?? 6379,
+            lazyConnect: false,
+            retryStrategy: (times) => Math.min(times * 100, 3000),
+            keepAlive: 10000,
+        };
+        if (options.password)
+            redisOpts.password = options.password;
+        this.redis = new ioredis_1.default(redisOpts);
+        this.redis.on('error', () => { });
+        this.keyPrefix = options.keyPrefix ?? PLATFORM_KEY_PREFIX;
+        this.userPlan = options.userPlan ?? 'free';
+        this.ttlManager = new ttl_manager_js_1.TtlManager(options.ttlConfig);
+        this.billingContext = options.billingContext;
+        this.allowUnscopedPurge = options.allowUnscopedPurge ?? false;
+    }
+    key(...parts) {
+        return this.keyPrefix + parts.join(':');
+    }
+    datasets() {
+        return new dataset_collection_client_js_1.DragonflyDatasetCollectionClient(this);
+    }
+    dataset(id) {
+        return new dataset_client_js_1.DragonflyDatasetClient(this, id);
+    }
+    keyValueStores() {
+        return new kv_store_collection_client_js_1.DragonflyKeyValueStoreCollectionClient(this);
+    }
+    keyValueStore(id) {
+        return new key_value_store_client_js_1.DragonflyKeyValueStoreClient(this, id);
+    }
+    requestQueues() {
+        return new rq_collection_client_js_1.DragonflyRequestQueueCollectionClient(this);
+    }
+    requestQueue(id, options) {
+        return new request_queue_client_js_1.DragonflyRequestQueueClient(this, id, options);
+    }
+    /**
+     * Purges storage, mirroring Crawlee's StorageClient.purge() contract: clear
+     * only the current run's default storages (dataset, KV store, request queue).
+     *
+     * Scoping rules — the default `crawlee:` prefix is shared by every user/run
+     * on the platform (datasets, request queues, billing, auth tokens all live
+     * under it), so purging the whole prefix would delete other tenants' data:
+     *
+     *  1. With `billingContext.runId`: resolve the run's storageIds from
+     *     `{prefix}run:{runId}` and delete only those three storage namespaces.
+     *  2. Without a run ID, but with a custom (scoped) `keyPrefix`: delete the
+     *     whole prefix — it is assumed dedicated to this client.
+     *  3. Without a run ID and with the shared default prefix: refuse, unless
+     *     `allowUnscopedPurge` was set explicitly (single-tenant opt-in).
+     *
+     * Deletion uses SCAN + batched UNLINK (see deleteByPattern) instead of a
+     * blocking KEYS walk followed by a single huge DEL.
+     */
+    async purge() {
+        const runId = this.billingContext?.runId;
+        if (runId && (await this.purgeRunDefaultStorages(runId)) !== null)
+            return;
+        if (this.keyPrefix === PLATFORM_KEY_PREFIX && !this.allowUnscopedPurge) {
+            throw new Error('DragonflyClient.purge() refused: keyPrefix "crawlee:" is shared by all users/runs ' +
+                "of the platform, so purging it would delete other tenants' data. Provide a " +
+                "billingContext.runId (purges only that run's default storages), pass a scoped " +
+                'keyPrefix, or set allowUnscopedPurge: true for single-tenant deployments.');
+        }
+        await this.deleteByPattern(`${this.keyPrefix}*`);
+    }
+    /**
+     * Delete the default storages (dataset / KV store / request queue) of a run,
+     * resolved from the run's metadata hash (`{prefix}run:{runId}` → storageIds).
+     *
+     * @returns the number of keys deleted, or null when run metadata is missing
+     *          (expired or never created) and nothing could be scoped.
+     */
+    async purgeRunDefaultStorages(runId) {
+        const data = await this.redis.hgetall(this.key('run', runId));
+        if (!data || !data.storageIds)
+            return null;
+        let storageIds;
+        try {
+            storageIds = JSON.parse(data.storageIds);
+        }
+        catch {
+            return null; // corrupt metadata — do not guess, fall back to prefix rules
+        }
+        let deleted = 0;
+        if (storageIds.dataset)
+            deleted += await this.deleteByPattern(this.key('dataset', storageIds.dataset, '*'));
+        if (storageIds.kvStore)
+            deleted += await this.deleteByPattern(this.key('kvs', storageIds.kvStore, '*'));
+        if (storageIds.requestQueue)
+            deleted += await this.deleteByPattern(this.key('rq', storageIds.requestQueue, '*'));
+        return deleted;
+    }
+    /**
+     * Incrementally delete every key matching `pattern` using SCAN + batched
+     * UNLINK. Unlike KEYS + DEL, this never walks the keyspace in one blocking
+     * pass, never materializes the full key list in a single reply, and never
+     * sends one gigantic delete command; UNLINK also reclaims memory
+     * asynchronously. Keys that vanish mid-scan are simply no-ops.
+     *
+     * @returns number of keys deleted (best effort).
+     */
+    async deleteByPattern(pattern, options) {
+        const scanCount = Math.max(1, options?.scanCount ?? DEFAULT_SCAN_COUNT);
+        const unlinkBatch = Math.max(1, options?.unlinkBatch ?? DEFAULT_UNLINK_BATCH);
+        let deleted = 0;
+        let cursor = '0';
+        do {
+            const [nextCursor, keys] = await this.redis.scan(cursor, 'MATCH', pattern, 'COUNT', scanCount);
+            cursor = nextCursor;
+            // A single SCAN reply can exceed COUNT, so chunk the UNLINKs too.
+            for (let i = 0; i < keys.length; i += unlinkBatch) {
+                const chunk = keys.slice(i, i + unlinkBatch);
+                await this.redis.unlink(...chunk);
+                deleted += chunk.length;
+            }
+        } while (cursor !== '0');
+        return deleted;
+    }
+    async teardown() {
+        // no-op: crawlee calls this during crawler.run() shutdown
+        // we manage the Redis connection lifecycle ourselves via disconnectDragonfly()
+    }
+    async disconnect() {
+        await this.redis.quit();
+    }
+    async setStatusMessage(message, options) {
+        const statusKey = this.key('status');
+        await this.redis.hset(statusKey, {
+            message,
+            level: options?.level ?? 'INFO',
+            isTerminal: options?.isStatusMessageTerminal ? '1' : '0',
+        });
+    }
+}
+exports.DragonflyClient = DragonflyClient;

package/dist/storage/index.d.ts

@@ -0,0 +1,11 @@
+export { DragonflyClient } from './dragonfly-client.js';
+export { TtlManager, type UserPlan, type StorageType, type TtlConfig } from './ttl-manager.js';
+export { DragonflyDatasetClient } from './dataset-client.js';
+export { DragonflyKeyValueStoreClient } from './key-value-store-client.js';
+export { DragonflyRequestQueueClient } from './request-queue-client.js';
+export { DragonflyDatasetCollectionClient } from './dataset-collection-client.js';
+export { DragonflyKeyValueStoreCollectionClient } from './kv-store-collection-client.js';
+export { DragonflyRequestQueueCollectionClient } from './rq-collection-client.js';
+export { RunMetadataManager, type RunMetadata, type RunStatus } from './run-metadata.js';
+export { LogManager, type LogConfig } from './log-manager.js';
+export { BillingTracker, type BillingSummary } from './billing-tracker.js';