scrapely 1.2.3 → 1.2.4

package/dist/storage/index.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BillingTracker = exports.LogManager = exports.RunMetadataManager = exports.DragonflyRequestQueueCollectionClient = exports.DragonflyKeyValueStoreCollectionClient = exports.DragonflyDatasetCollectionClient = exports.DragonflyRequestQueueClient = exports.DragonflyKeyValueStoreClient = exports.DragonflyDatasetClient = exports.TtlManager = exports.DragonflyClient = void 0;
+var dragonfly_client_js_1 = require("./dragonfly-client.js");
+Object.defineProperty(exports, "DragonflyClient", { enumerable: true, get: function () { return dragonfly_client_js_1.DragonflyClient; } });
+var ttl_manager_js_1 = require("./ttl-manager.js");
+Object.defineProperty(exports, "TtlManager", { enumerable: true, get: function () { return ttl_manager_js_1.TtlManager; } });
+var dataset_client_js_1 = require("./dataset-client.js");
+Object.defineProperty(exports, "DragonflyDatasetClient", { enumerable: true, get: function () { return dataset_client_js_1.DragonflyDatasetClient; } });
+var key_value_store_client_js_1 = require("./key-value-store-client.js");
+Object.defineProperty(exports, "DragonflyKeyValueStoreClient", { enumerable: true, get: function () { return key_value_store_client_js_1.DragonflyKeyValueStoreClient; } });
+var request_queue_client_js_1 = require("./request-queue-client.js");
+Object.defineProperty(exports, "DragonflyRequestQueueClient", { enumerable: true, get: function () { return request_queue_client_js_1.DragonflyRequestQueueClient; } });
+var dataset_collection_client_js_1 = require("./dataset-collection-client.js");
+Object.defineProperty(exports, "DragonflyDatasetCollectionClient", { enumerable: true, get: function () { return dataset_collection_client_js_1.DragonflyDatasetCollectionClient; } });
+var kv_store_collection_client_js_1 = require("./kv-store-collection-client.js");
+Object.defineProperty(exports, "DragonflyKeyValueStoreCollectionClient", { enumerable: true, get: function () { return kv_store_collection_client_js_1.DragonflyKeyValueStoreCollectionClient; } });
+var rq_collection_client_js_1 = require("./rq-collection-client.js");
+Object.defineProperty(exports, "DragonflyRequestQueueCollectionClient", { enumerable: true, get: function () { return rq_collection_client_js_1.DragonflyRequestQueueCollectionClient; } });
+var run_metadata_js_1 = require("./run-metadata.js");
+Object.defineProperty(exports, "RunMetadataManager", { enumerable: true, get: function () { return run_metadata_js_1.RunMetadataManager; } });
+var log_manager_js_1 = require("./log-manager.js");
+Object.defineProperty(exports, "LogManager", { enumerable: true, get: function () { return log_manager_js_1.LogManager; } });
+var billing_tracker_js_1 = require("./billing-tracker.js");
+Object.defineProperty(exports, "BillingTracker", { enumerable: true, get: function () { return billing_tracker_js_1.BillingTracker; } });

package/dist/storage/key-value-store-client.d.ts

@@ -0,0 +1,61 @@
+import type { KeyValueStoreClient, KeyValueStoreInfo, KeyValueStoreRecord, KeyValueStoreClientGetRecordOptions, KeyValueStoreClientListOptions, KeyValueStoreClientUpdateOptions, KeyValueStoreClientListData, KeyValueStoreItemData, KeyValueStoreRecordOptions } from '@crawlee/types';
+import type { DragonflyClient } from './dragonfly-client.js';
+/**
+ * Type for the iteratee function used in forEachKey
+ */
+export type KeyConsumer = (key: string, index: number, info: {
+    size: number;
+}) => Promise<void> | void;
+/**
+ * Options for forEachKey iteration
+ */
+export interface ForEachKeyOptions extends KeyValueStoreClientListOptions {
+}
+export declare class DragonflyKeyValueStoreClient implements KeyValueStoreClient {
+    private client;
+    private id;
+    private infoKey;
+    private recordsKey;
+    private metaKey;
+    constructor(client: DragonflyClient, id: string);
+    private estimateStoredRecordBytes;
+    get(): Promise<KeyValueStoreInfo | undefined>;
+    update(newFields: KeyValueStoreClientUpdateOptions): Promise<Partial<KeyValueStoreInfo>>;
+    delete(): Promise<void>;
+    recordExists(key: string): Promise<boolean>;
+    getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;
+    setRecord(record: KeyValueStoreRecord, _options?: KeyValueStoreRecordOptions, billingContext?: {
+        runId: string;
+        userId: string;
+    }): Promise<void>;
+    deleteRecord(key: string): Promise<void>;
+    listKeys(options?: KeyValueStoreClientListOptions): Partial<AsyncIterable<KeyValueStoreItemData>> & Promise<KeyValueStoreClientListData>;
+    private _listKeys;
+    keys(options?: KeyValueStoreClientListOptions): AsyncIterable<string> & Promise<KeyValueStoreClientListData>;
+    values(options?: KeyValueStoreClientListOptions): AsyncIterable<unknown> & Promise<unknown[]>;
+    entries(options?: KeyValueStoreClientListOptions): AsyncIterable<[string, unknown]> & Promise<[string, unknown][]>;
+    /**
+     * Iterates over all keys in the key-value store, calling the iteratee function for each key.
+     * Similar to Apify's KeyValueStore.forEachKey() method.
+     */
+    forEachKey(iteratee: KeyConsumer, options?: ForEachKeyOptions): Promise<void>;
+    /**
+     * Gets a value from the key-value store that is automatically saved when modified.
+     * If the key doesn't exist, returns the defaultValue and stores it for future use.
+     * Similar to Apify's KeyValueStore.getAutoSavedValue() method.
+     *
+     * @param key - The key of the record to get
+     * @param defaultValue - The default value to return and store if the key doesn't exist
+     * @returns The stored value or the defaultValue
+     */
+    getAutoSavedValue<T>(key: string, defaultValue: T): Promise<T>;
+    /**
+     * Returns a URL for the given key that may be used to publicly access the value
+     * in the remote key-value store.
+     * Similar to Apify's KeyValueStore.getPublicUrl() method.
+     *
+     * @param key - The key of the record to get the public URL for
+     * @returns A public URL for accessing the record
+     */
+    getPublicUrl(key: string): string;
+}

package/dist/storage/key-value-store-client.js

@@ -0,0 +1,306 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DragonflyKeyValueStoreClient = void 0;
+const node_stream_1 = require("node:stream");
+const utils_js_1 = require("./utils.js");
+const billing_tracker_js_1 = require("./billing-tracker.js");
+const configuration_js_1 = require("../configuration.js");
+const kvs_iteration_js_1 = require("./kvs-iteration.js");
+function makeAsyncIterableListKeys(promiseFn) {
+    const promise = promiseFn();
+    const iterable = {
+        [Symbol.asyncIterator]: () => {
+            let items = null;
+            let index = 0;
+            return {
+                async next() {
+                    if (!items)
+                        items = (await promise).items;
+                    if (index >= items.length)
+                        return { done: true, value: undefined };
+                    return { done: false, value: items[index++] };
+                },
+            };
+        },
+        then: promise.then.bind(promise),
+        catch: promise.catch.bind(promise),
+        finally: promise.finally.bind(promise),
+        [Symbol.toStringTag]: 'AsyncIterablePromise',
+    };
+    return iterable;
+}
+class DragonflyKeyValueStoreClient {
+    client;
+    id;
+    infoKey;
+    recordsKey;
+    metaKey;
+    constructor(client, id) {
+        this.client = client;
+        this.id = id;
+        this.infoKey = client.key('kvs', id, 'info');
+        this.recordsKey = client.key('kvs', id, 'records');
+        this.metaKey = client.key('kvs', id, 'meta');
+    }
+    estimateStoredRecordBytes(key, stored, contentType) {
+        if (stored === null)
+            return 0;
+        if (contentType === 'application/octet-stream') {
+            return billing_tracker_js_1.BillingTracker.estimateRecordBytes(key, Buffer.from(stored, 'base64'));
+        }
+        return billing_tracker_js_1.BillingTracker.estimateRecordBytes(key, stored);
+    }
+    async get() {
+        const info = await this.client.redis.hgetall(this.infoKey);
+        if (!info || !info.id)
+            return undefined;
+        return {
+            id: info.id,
+            name: info.name || undefined,
+            userId: info.userId,
+            createdAt: new Date(info.createdAt),
+            modifiedAt: new Date(info.modifiedAt),
+            accessedAt: new Date(info.accessedAt),
+        };
+    }
+    async update(newFields) {
+        if (newFields.name !== undefined) {
+            await this.client.redis.hset(this.infoKey, 'name', newFields.name);
+            const indexKey = this.client.key('kvs:index');
+            const raw = await this.client.redis.hget(indexKey, `id:${this.id}`);
+            if (raw) {
+                const data = JSON.parse(raw);
+                if (data.name)
+                    await this.client.redis.hdel(indexKey, `name:${data.name}`);
+                data.name = newFields.name;
+                await this.client.redis.hset(indexKey, `id:${this.id}`, JSON.stringify(data));
+                await this.client.redis.hset(indexKey, `name:${newFields.name}`, JSON.stringify(data));
+            }
+        }
+        await this.client.redis.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+        return { name: newFields.name };
+    }
+    async delete() {
+        const info = await this.client.redis.hgetall(this.infoKey);
+        await this.client.redis.del(this.infoKey, this.recordsKey, this.metaKey);
+        const indexKey = this.client.key('kvs:index');
+        await this.client.redis.hdel(indexKey, `id:${this.id}`);
+        if (info?.name)
+            await this.client.redis.hdel(indexKey, `name:${info.name}`);
+    }
+    async recordExists(key) {
+        return (await this.client.redis.hexists(this.recordsKey, key)) === 1;
+    }
+    async getRecord(key, options) {
+        const raw = await this.client.redis.hget(this.recordsKey, key);
+        if (raw === null)
+            return undefined;
+        // Track read for billing - increment both user-level and info hash counters
+        if (this.client.billingContext) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            await billingTracker.trackRead(this.client.billingContext.userId, 1);
+            await this.client.redis.hincrby(this.infoKey, 'readCount', 1);
+            // Track internal data transfer (bytes returned to actor)
+            const bytesTransferred = Buffer.byteLength(raw, 'utf8');
+            await billingTracker.trackInternalTransfer(this.client.billingContext.runId, this.client.billingContext.userId, bytesTransferred);
+            // Register this store as accessed for per-run aggregation
+            await this.client.redis.sadd(this.client.key('run', this.client.billingContext.runId, 'kvs_accessed'), this.id);
+        }
+        const meta = await this.client.redis.hget(this.metaKey, key);
+        const contentType = meta ? JSON.parse(meta).contentType : undefined;
+        const isBinary = contentType === 'application/octet-stream';
+        let value;
+        if (isBinary) {
+            value = Buffer.from(raw, 'base64');
+        }
+        else if (options?.buffer) {
+            value = Buffer.from(raw);
+        }
+        else {
+            try {
+                value = JSON.parse(raw);
+            }
+            catch {
+                value = raw;
+            }
+        }
+        if (options?.stream) {
+            const buf = Buffer.isBuffer(value) ? value : Buffer.from(typeof value === 'string' ? value : JSON.stringify(value));
+            value = node_stream_1.Readable.from(buf);
+        }
+        return { key, value, contentType };
+    }
+    async setRecord(record, _options, billingContext) {
+        const { key, value, contentType } = record;
+        let stored;
+        let billingValue = value; // Use for billing estimation (may differ from original if stream)
+        // Handle Readable streams (e.g., fs.createReadStream) - Apify compatibility
+        // Streams are objects with a pipe method. We need to buffer them.
+        if (value && typeof value === 'object' && typeof value.pipe === 'function') {
+            const chunks = [];
+            for await (const chunk of value) {
+                chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+            }
+            const buffered = Buffer.concat(chunks);
+            stored = buffered.toString('base64');
+            billingValue = buffered; // Use buffered value for billing estimation
+        }
+        else if (Buffer.isBuffer(value)) {
+            stored = value.toString('base64');
+        }
+        else if (typeof value === 'string') {
+            stored = value;
+        }
+        else {
+            stored = JSON.stringify(value);
+        }
+        const [existingRaw, existingMetaRaw] = await Promise.all([
+            this.client.redis.hget(this.recordsKey, key),
+            this.client.redis.hget(this.metaKey, key),
+        ]);
+        const existingContentType = existingMetaRaw ? JSON.parse(existingMetaRaw).contentType : undefined;
+        const previousBytes = this.estimateStoredRecordBytes(key, existingRaw, existingContentType);
+        const nextBytes = billing_tracker_js_1.BillingTracker.estimateRecordBytes(key, billingValue);
+        const deltaBytes = nextBytes - previousBytes;
+        const pipeline = this.client.redis.pipeline();
+        pipeline.hset(this.recordsKey, key, stored);
+        if (contentType) {
+            pipeline.hset(this.metaKey, key, JSON.stringify({ contentType }));
+        }
+        pipeline.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+        await pipeline.exec();
+        // Track billing using client's billing context
+        if (this.client.billingContext) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            if (deltaBytes !== 0) {
+                await billingTracker.trackStorageBytes(this.client.billingContext.runId, this.client.billingContext.userId, deltaBytes);
+            }
+            await billingTracker.trackWrite(this.client.billingContext.userId);
+            await this.client.redis.hincrby(this.infoKey, 'writeCount', 1);
+            // Track internal data transfer (bytes written to storage)
+            await billingTracker.trackInternalTransfer(this.client.billingContext.runId, this.client.billingContext.userId, nextBytes);
+            // Register this store as accessed for per-run aggregation
+            await this.client.redis.sadd(this.client.key('run', this.client.billingContext.runId, 'kvs_accessed'), this.id);
+        }
+    }
+    async deleteRecord(key) {
+        const [existingRaw, existingMetaRaw] = await Promise.all([
+            this.client.redis.hget(this.recordsKey, key),
+            this.client.redis.hget(this.metaKey, key),
+        ]);
+        const pipeline = this.client.redis.pipeline();
+        pipeline.hdel(this.recordsKey, key);
+        pipeline.hdel(this.metaKey, key);
+        pipeline.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+        await pipeline.exec();
+        if (this.client.billingContext && existingRaw !== null) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            const existingContentType = existingMetaRaw ? JSON.parse(existingMetaRaw).contentType : undefined;
+            const releasedBytes = this.estimateStoredRecordBytes(key, existingRaw, existingContentType);
+            if (releasedBytes > 0) {
+                await billingTracker.trackStorageBytes(this.client.billingContext.runId, this.client.billingContext.userId, -releasedBytes);
+            }
+        }
+    }
+    listKeys(options) {
+        return makeAsyncIterableListKeys(() => this._listKeys(options));
+    }
+    async _listKeys(options) {
+        // HKEYS returns only field names; HGETALL would transfer every record's
+        // value (including base64-encoded blobs) just to discard them here.
+        let allKeys = await this.client.redis.hkeys(this.recordsKey);
+        if (options?.prefix) {
+            allKeys = allKeys.filter(k => k.startsWith(options.prefix));
+        }
+        if (options?.collection) {
+            const info = await this.client.redis.hgetall(this.infoKey);
+            allKeys = (0, kvs_iteration_js_1.filterKeysByCollection)(allKeys, info.schema, options.collection);
+        }
+        allKeys.sort();
+        if (options?.exclusiveStartKey) {
+            // Keys are sorted, so keep everything strictly after the start key.
+            // Comparing (instead of indexOf) also handles the start key having been
+            // deleted between pages, which previously restarted the page from the
+            // beginning and could make paginating iterators loop forever.
+            allKeys = allKeys.filter((k) => k > options.exclusiveStartKey);
+        }
+        const limit = options?.limit ?? allKeys.length;
+        const isTruncated = allKeys.length > limit;
+        const sliced = allKeys.slice(0, limit);
+        // Fetch value lengths only (HSTRLEN) in one pipelined round trip;
+        // this avoids transferring each record's full value just to compute its size.
+        const pipeline = this.client.redis.pipeline();
+        for (const key of sliced)
+            pipeline.hstrlen(this.recordsKey, key);
+        const results = await pipeline.exec();
+        const items = sliced.map((key, i) => ({
+            key,
+            size: results?.[i]?.[1] ?? 0,
+        }));
+        // Track list operation for billing
+        if (this.client.billingContext) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            await billingTracker.trackList(this.client.billingContext.userId, 1);
+            // Increment the per-store listCount so the worker can tally KVS list operations
+            await this.client.redis.hincrby(this.infoKey, 'listCount', 1);
+            // Register this store as accessed for per-run aggregation (always, even if empty)
+            await this.client.redis.sadd(this.client.key('run', this.client.billingContext.runId, 'kvs_accessed'), this.id);
+        }
+        return {
+            count: items.length,
+            limit,
+            exclusiveStartKey: options?.exclusiveStartKey,
+            isTruncated,
+            nextExclusiveStartKey: isTruncated ? sliced[sliced.length - 1] : undefined,
+            items,
+        };
+    }
+    keys(options) {
+        return (0, kvs_iteration_js_1.makePaginatingKeysIterable)((opts) => this._listKeys(opts), options);
+    }
+    values(options) {
+        return (0, kvs_iteration_js_1.makePaginatingValuesIterable)((opts) => this._listKeys(opts), (key) => this.getRecord(key), options);
+    }
+    entries(options) {
+        return (0, kvs_iteration_js_1.makePaginatingEntriesIterable)((opts) => this._listKeys(opts), (key) => this.getRecord(key), options);
+    }
+    /**
+     * Iterates over all keys in the key-value store, calling the iteratee function for each key.
+     * Similar to Apify's KeyValueStore.forEachKey() method.
+     */
+    async forEachKey(iteratee, options) {
+        return (0, kvs_iteration_js_1.forEachKeyPaginated)((opts) => this._listKeys(opts), iteratee, options);
+    }
+    /**
+     * Gets a value from the key-value store that is automatically saved when modified.
+     * If the key doesn't exist, returns the defaultValue and stores it for future use.
+     * Similar to Apify's KeyValueStore.getAutoSavedValue() method.
+     *
+     * @param key - The key of the record to get
+     * @param defaultValue - The default value to return and store if the key doesn't exist
+     * @returns The stored value or the defaultValue
+     */
+    async getAutoSavedValue(key, defaultValue) {
+        const record = await this.getRecord(key);
+        if (record === undefined) {
+            // Store the default value and return it
+            await this.setRecord({ key, value: defaultValue });
+            return defaultValue;
+        }
+        return record.value;
+    }
+    /**
+     * Returns a URL for the given key that may be used to publicly access the value
+     * in the remote key-value store.
+     * Similar to Apify's KeyValueStore.getPublicUrl() method.
+     *
+     * @param key - The key of the record to get the public URL for
+     * @returns A public URL for accessing the record
+     */
+    getPublicUrl(key) {
+        const config = configuration_js_1.Configuration.getGlobalConfig();
+        const baseUrl = config.get('apiPublicBaseUrl') || config.get('apiBaseUrl') || 'https://api-staging.scrape.ly';
+        return `${baseUrl}/v2/key-value-stores/${this.id}/records/${key}`;
+    }
+}
+exports.DragonflyKeyValueStoreClient = DragonflyKeyValueStoreClient;

package/dist/storage/kv-store-collection-client.d.ts

@@ -0,0 +1,8 @@
+import type { KeyValueStoreCollectionClient, KeyValueStoreInfo, PaginatedList } from '@crawlee/types';
+import type { DragonflyClient } from './dragonfly-client.js';
+export declare class DragonflyKeyValueStoreCollectionClient implements KeyValueStoreCollectionClient {
+    private client;
+    constructor(client: DragonflyClient);
+    getOrCreate(name?: string): Promise<KeyValueStoreInfo>;
+    list(): Promise<PaginatedList<KeyValueStoreInfo>>;
+}

package/dist/storage/kv-store-collection-client.js

@@ -0,0 +1,76 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DragonflyKeyValueStoreCollectionClient = void 0;
+const utils_js_1 = require("./utils.js");
+class DragonflyKeyValueStoreCollectionClient {
+    client;
+    constructor(client) {
+        this.client = client;
+    }
+    async getOrCreate(name) {
+        const indexKey = this.client.key('kvs:index');
+        if (name) {
+            const existing = await this.client.redis.hget(indexKey, `name:${name}`);
+            if (existing)
+                return JSON.parse(existing);
+        }
+        const id = (0, utils_js_1.generateId)();
+        const now = (0, utils_js_1.nowISO)();
+        const data = {
+            id,
+            name,
+            userId: '1',
+            createdAt: now,
+            modifiedAt: now,
+            accessedAt: now,
+        };
+        await this.client.redis.hset(indexKey, `id:${id}`, JSON.stringify(data));
+        if (name)
+            await this.client.redis.hset(indexKey, `name:${name}`, JSON.stringify(data));
+        const infoKey = this.client.key('kvs', id, 'info');
+        const recordsKey = this.client.key('kvs', id, 'records');
+        const metaKey = this.client.key('kvs', id, 'meta');
+        await this.client.redis.hset(infoKey, {
+            id,
+            name: name ?? '',
+            userId: '1',
+            createdAt: now.toISOString(),
+            modifiedAt: now.toISOString(),
+            accessedAt: now.toISOString(),
+        });
+        // Set TTL for unnamed KV stores (named stores have indefinite retention)
+        if (!name) {
+            const ttl = this.client.ttlManager.getTtl('kv', this.client.userPlan, false);
+            if (ttl) {
+                await this.client.ttlManager.setExpiry(this.client.redis, [infoKey, recordsKey, metaKey], ttl);
+            }
+        }
+        return data;
+    }
+    async list() {
+        const indexKey = this.client.key('kvs:index');
+        const all = await this.client.redis.hgetall(indexKey);
+        const items = [];
+        const seen = new Set();
+        for (const [key, val] of Object.entries(all)) {
+            if (!key.startsWith('id:'))
+                continue;
+            const parsed = JSON.parse(val);
+            if (seen.has(parsed.id))
+                continue;
+            seen.add(parsed.id);
+            const infoKey = this.client.key('kvs', parsed.id, 'info');
+            // Skip and clean up stale index entries whose keys have expired via TTL
+            const exists = await this.client.redis.exists(infoKey);
+            if (!exists) {
+                await this.client.redis.hdel(indexKey, `id:${parsed.id}`);
+                if (parsed.name)
+                    await this.client.redis.hdel(indexKey, `name:${parsed.name}`);
+                continue;
+            }
+            items.push(parsed);
+        }
+        return { total: items.length, count: items.length, offset: 0, limit: items.length, items };
+    }
+}
+exports.DragonflyKeyValueStoreCollectionClient = DragonflyKeyValueStoreCollectionClient;

package/dist/storage/kvs-iteration.d.ts

@@ -0,0 +1,17 @@
+import type { KeyValueStoreClientListData, KeyValueStoreClientListOptions, KeyValueStoreItemData } from '@crawlee/types';
+export type ListKeysFn = (options?: KeyValueStoreClientListOptions) => Promise<KeyValueStoreClientListData>;
+/**
+ * Filter keys by Apify-style collection config from a KVS schema.
+ */
+export declare function filterKeysByCollection(allKeys: string[], schemaJson: string | undefined, collection: string): string[];
+export declare function paginateKeyItems(listKeys: ListKeysFn, options?: KeyValueStoreClientListOptions): AsyncGenerator<KeyValueStoreItemData>;
+export declare function forEachKeyPaginated(listKeys: ListKeysFn, iteratee: (key: string, index: number, info: {
+    size: number;
+}) => Promise<void> | void, options?: KeyValueStoreClientListOptions, startIndex?: number): Promise<void>;
+export declare function makePaginatingKeysIterable(listKeys: ListKeysFn, options?: KeyValueStoreClientListOptions): AsyncIterable<string> & Promise<KeyValueStoreClientListData>;
+export declare function makePaginatingValuesIterable(listKeys: ListKeysFn, getRecord: (key: string) => Promise<{
+    value: unknown;
+} | undefined>, options?: KeyValueStoreClientListOptions): AsyncIterable<unknown> & Promise<unknown[]>;
+export declare function makePaginatingEntriesIterable(listKeys: ListKeysFn, getRecord: (key: string) => Promise<{
+    value: unknown;
+} | undefined>, options?: KeyValueStoreClientListOptions): AsyncIterable<[string, unknown]> & Promise<[string, unknown][]>;

package/dist/storage/kvs-iteration.js

@@ -0,0 +1,126 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.filterKeysByCollection = filterKeysByCollection;
+exports.paginateKeyItems = paginateKeyItems;
+exports.forEachKeyPaginated = forEachKeyPaginated;
+exports.makePaginatingKeysIterable = makePaginatingKeysIterable;
+exports.makePaginatingValuesIterable = makePaginatingValuesIterable;
+exports.makePaginatingEntriesIterable = makePaginatingEntriesIterable;
+/**
+ * Filter keys by Apify-style collection config from a KVS schema.
+ */
+function filterKeysByCollection(allKeys, schemaJson, collection) {
+    if (!schemaJson) {
+        throw new Error('Key-value store schema is not configured for collection filtering.');
+    }
+    let schema;
+    try {
+        schema = JSON.parse(schemaJson);
+    }
+    catch {
+        throw new Error('Key-value store schema is invalid JSON.');
+    }
+    const cfg = schema?.collections?.[collection];
+    if (!cfg) {
+        throw new Error(`Collection '${collection}' is not defined in the key-value store schema.`);
+    }
+    if (cfg.keyPrefix && typeof cfg.keyPrefix === 'string') {
+        return allKeys.filter((k) => k.startsWith(cfg.keyPrefix));
+    }
+    if (cfg.key && typeof cfg.key === 'string') {
+        return allKeys.filter((k) => k === cfg.key);
+    }
+    if (Array.isArray(cfg.keys)) {
+        const set = new Set(cfg.keys.map(String));
+        return allKeys.filter((k) => set.has(k));
+    }
+    return allKeys;
+}
+async function* paginateKeyItems(listKeys, options = {}) {
+    let exclusiveStartKey = options.exclusiveStartKey;
+    const { prefix, collection, limit } = options;
+    for (;;) {
+        const response = await listKeys({ prefix, collection, limit, exclusiveStartKey });
+        for (const item of response.items) {
+            yield item;
+        }
+        if (!response.isTruncated || !response.nextExclusiveStartKey) {
+            break;
+        }
+        exclusiveStartKey = response.nextExclusiveStartKey;
+    }
+}
+async function forEachKeyPaginated(listKeys, iteratee, options = {}, startIndex = 0) {
+    let index = startIndex;
+    let exclusiveStartKey = options.exclusiveStartKey;
+    const { prefix, collection, limit } = options;
+    for (;;) {
+        const response = await listKeys({ prefix, collection, limit, exclusiveStartKey });
+        for (const item of response.items) {
+            await iteratee(item.key, index++, { size: item.size });
+        }
+        if (!response.isTruncated || !response.nextExclusiveStartKey) {
+            return;
+        }
+        exclusiveStartKey = response.nextExclusiveStartKey;
+    }
+}
+function makeHybridIterable(iterate, promise) {
+    // A native Promise has no own enumerable properties (then/catch/finally live on
+    // Promise.prototype), so Object.assign(iterable, promise) copies nothing and the
+    // result is not thenable — awaiting it resolved to the iterable object itself.
+    // Explicitly bind the promise methods instead (same pattern as
+    // makeAsyncIterableListKeys in key-value-store-client.ts).
+    const iterable = {
+        [Symbol.asyncIterator]: iterate,
+        then: promise.then.bind(promise),
+        catch: promise.catch.bind(promise),
+        finally: promise.finally.bind(promise),
+        [Symbol.toStringTag]: 'AsyncIterablePromise',
+    };
+    return iterable;
+}
+function makePaginatingKeysIterable(listKeys, options) {
+    const opts = options ?? {};
+    return makeHybridIterable(() => (async function* () {
+        for await (const item of paginateKeyItems(listKeys, opts)) {
+            yield item.key;
+        }
+    })(), listKeys(opts));
+}
+function makePaginatingValuesIterable(listKeys, getRecord, options) {
+    const opts = options ?? {};
+    return makeHybridIterable(() => (async function* () {
+        for await (const item of paginateKeyItems(listKeys, opts)) {
+            const record = await getRecord(item.key);
+            if (record)
+                yield record.value;
+        }
+    })(), (async () => {
+        const values = [];
+        for await (const item of paginateKeyItems(listKeys, opts)) {
+            const record = await getRecord(item.key);
+            if (record)
+                values.push(record.value);
+        }
+        return values;
+    })());
+}
+function makePaginatingEntriesIterable(listKeys, getRecord, options) {
+    const opts = options ?? {};
+    return makeHybridIterable(() => (async function* () {
+        for await (const item of paginateKeyItems(listKeys, opts)) {
+            const record = await getRecord(item.key);
+            if (record)
+                yield [item.key, record.value];
+        }
+    })(), (async () => {
+        const entries = [];
+        for await (const item of paginateKeyItems(listKeys, opts)) {
+            const record = await getRecord(item.key);
+            if (record)
+                entries.push([item.key, record.value]);
+        }
+        return entries;
+    })());
+}