scrapely 1.2.3 → 1.2.5

package/dist/storage/api-storage-client.js

@@ -0,0 +1,910 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.ApiStorageClientError = void 0;
+const node_crypto_1 = require("node:crypto");
+const kvs_iteration_js_1 = require("./kvs-iteration.js");
+function buildRqQueryParams(baseClientKey, options) {
+    const params = new URLSearchParams();
+    params.set('clientKey', options?.clientKey ?? baseClientKey);
+    if (options?.forefront)
+        params.set('forefront', 'true');
+    if (options?.lockSecs !== undefined)
+        params.set('lockSecs', String(options.lockSecs));
+    if (options?.limit !== undefined)
+        params.set('limit', String(options.limit));
+    if (options?.runId)
+        params.set('runId', options.runId);
+    const query = params.toString();
+    return query ? `?${query}` : '';
+}
+function mapRequestQueueInfo(id, result) {
+    return {
+        id: result.id ?? id,
+        name: result.name ?? undefined,
+        userId: result.userId ?? undefined,
+        createdAt: parseApiDate(result.createdAt),
+        modifiedAt: parseApiDate(result.modifiedAt),
+        accessedAt: parseApiDate(result.accessedAt),
+        expireAt: result.expireAt,
+        totalRequestCount: result.totalRequestCount,
+        handledRequestCount: result.handledRequestCount,
+        pendingRequestCount: result.pendingRequestCount,
+        hadMultipleClients: result.hadMultipleClients,
+        actId: result.actId ?? result.actorId,
+        actRunId: result.actRunId,
+        stats: result.stats,
+    };
+}
+/**
+ * Adaptive rate limiter that tracks server-provided rate limit headers.
+ * Coordinates across all runs for a user by reading X-RateLimit-* headers.
+ */
+class AdaptiveRateLimiter {
+    limit = 1000;
+    remaining = 1000;
+    resetAt = 0;
+    lowWaterMark = 50; // Start throttling when remaining < this
+    minDelay = 50; // Minimum delay between requests (ms)
+    maxDelay = 1000; // Maximum delay when rate limited (ms)
+    lastRequestTime = 0;
+    /**
+     * Update internal state from response headers.
+     * Called after each successful API response.
+     */
+    updateFromHeaders(headers) {
+        const limitHeader = headers.get('X-RateLimit-Limit');
+        const remainingHeader = headers.get('X-RateLimit-Remaining');
+        const resetHeader = headers.get('X-RateLimit-Reset');
+        if (limitHeader)
+            this.limit = parseInt(limitHeader, 10) || 600;
+        if (remainingHeader)
+            this.remaining = parseInt(remainingHeader, 10) || 0;
+        if (resetHeader)
+            this.resetAt = parseInt(resetHeader, 10) * 1000; // Convert to ms
+    }
+    /**
+     * Wait before making the next request if needed.
+     * Uses adaptive throttling based on remaining capacity.
+     */
+    async waitForSlot() {
+        const now = Date.now();
+        // If we're past the reset time, reset our tracking
+        if (now >= this.resetAt) {
+            this.remaining = this.limit;
+        }
+        // Calculate adaptive delay based on remaining capacity
+        let delay = this.minDelay;
+        if (this.remaining < this.lowWaterMark) {
+            // We're running low - slow down proportionally
+            const ratio = this.remaining / this.lowWaterMark;
+            delay = Math.min(this.maxDelay, this.minDelay + (1 - ratio) * 500);
+        }
+        else if (this.remaining < this.limit * 0.2) {
+            // Under 20% capacity - moderate slowdown
+            delay = 100;
+        }
+        // Ensure minimum time between requests
+        const timeSinceLastRequest = now - this.lastRequestTime;
+        if (timeSinceLastRequest < delay) {
+            await new Promise(resolve => setTimeout(resolve, delay - timeSinceLastRequest));
+        }
+        this.lastRequestTime = Date.now();
+    }
+    /**
+     * Handle a 429 rate limit response.
+     * Parses Retry-After header and waits the appropriate time.
+     */
+    async handleRateLimited(response) {
+        const retryAfter = response.headers.get('Retry-After');
+        let waitMs = 1000; // Default 1 second
+        if (retryAfter) {
+            // Retry-After can be seconds or a date
+            const seconds = parseInt(retryAfter, 10);
+            if (!isNaN(seconds)) {
+                waitMs = seconds * 1000;
+            }
+            else {
+                // It's a date string
+                const retryDate = new Date(retryAfter).getTime();
+                if (!isNaN(retryDate)) {
+                    waitMs = Math.max(0, retryDate - Date.now());
+                }
+            }
+        }
+        // Add jitter to prevent thundering herd
+        waitMs = waitMs + Math.random() * 100;
+        console.warn(`[RateLimiter] Rate limited, waiting ${Math.round(waitMs)}ms before retry`);
+        await new Promise(resolve => setTimeout(resolve, waitMs));
+    }
+}
+/**
+ * Unwrap Apify-style v2 response envelopes ({ data: ... }).
+ */
+function unwrapData(value) {
+    if (Array.isArray(value))
+        return value;
+    if (value && typeof value === 'object' && 'data' in value) {
+        return value.data;
+    }
+    return value;
+}
+function parseApiDate(value) {
+    if (value instanceof Date)
+        return value;
+    if (typeof value === 'string' && value) {
+        const parsed = new Date(value);
+        if (!isNaN(parsed.getTime()))
+            return parsed;
+    }
+    return new Date();
+}
+const STORAGE_UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+/** Crawlee opens by name or id; only UUIDs are valid API resource ids. */
+function isStorageUuid(id) {
+    return STORAGE_UUID_REGEX.test(id);
+}
+class ApiStorageClientError extends Error {
+    statusCode;
+    constructor(message, statusCode) {
+        super(message);
+        this.name = 'ApiStorageClientError';
+        this.statusCode = statusCode;
+    }
+}
+exports.ApiStorageClientError = ApiStorageClientError;
+function isApiStorageClientError(err) {
+    return err instanceof ApiStorageClientError;
+}
+/** Map GET storage errors: rethrow 403, return undefined for 404 (triggers getOrCreate for names only). */
+function handleStorageGetError(err) {
+    if (isApiStorageClientError(err)) {
+        if (err.statusCode === 403)
+            throw err;
+        if (err.statusCode === 404)
+            return undefined;
+    }
+    throw err;
+}
+function storageCollectionHeaders(config) {
+    return {
+        Authorization: `Bearer ${config.token}`,
+        'X-Scrapely-Run-Id': config.runId,
+        'Content-Type': 'application/json',
+    };
+}
+/** POST /v2/{datasets|key-value-stores|request-queues} to create or get-by-name. */
+async function postStorageCollection(config, path, name) {
+    const query = name ? `?name=${encodeURIComponent(name)}` : '';
+    const url = `${config.apiUrl}/v2${path}${query}`;
+    const response = await fetch(url, {
+        method: 'POST',
+        headers: storageCollectionHeaders(config),
+    });
+    if (!response.ok) {
+        const error = await response.json().catch(() => ({ error: { message: response.statusText } }));
+        throw new Error(error.error?.message || `Failed to create storage: ${response.status}`);
+    }
+    const body = await response.json();
+    return unwrapData(body);
+}
+/**
+ * Helper to create AsyncIterable + Promise result (matches Crawlee pattern)
+ */
+function makeAsyncIterableResult(promiseFn) {
+    const promise = promiseFn();
+    const iterable = {
+        [Symbol.asyncIterator]: () => {
+            let items = null;
+            let index = 0;
+            return {
+                async next() {
+                    if (!items)
+                        items = (await promise).items;
+                    if (index >= items.length)
+                        return { done: true, value: undefined };
+                    return { done: false, value: items[index++] };
+                },
+            };
+        },
+        then: promise.then.bind(promise),
+        catch: promise.catch.bind(promise),
+        finally: promise.finally.bind(promise),
+        [Symbol.toStringTag]: 'AsyncIterablePromise',
+    };
+    return iterable;
+}
+/**
+ * Base class for API-based storage clients.
+ * Communicates with internal storage API instead of direct DB access.
+ * No sensitive credentials are stored - only the user's API key.
+ */
+class BaseApiClient {
+    config;
+    headers;
+    // Shared rate limiter instance - coordinates across all API clients
+    // This is static so it's shared across all storage clients for this run
+    static rateLimiter = new AdaptiveRateLimiter();
+    // Maximum retries for rate-limited requests
+    static maxRetries = 3;
+    // Default timeout for requests (30 seconds)
+    static defaultTimeoutMs = 30000;
+    constructor(config) {
+        this.config = config;
+        this.headers = {
+            'Authorization': `Bearer ${config.token}`,
+            'X-Scrapely-Run-Id': config.runId,
+            'Content-Type': 'application/json',
+        };
+    }
+    async request(method, path, body, contentType, options) {
+        const url = `${this.config.apiUrl}/v2${path}`;
+        const headers = { ...this.headers };
+        if (contentType) {
+            headers['Content-Type'] = contentType;
+        }
+        const timeoutMs = options?.timeoutSecs
+            ? options.timeoutSecs * 1000
+            : BaseApiClient.defaultTimeoutMs;
+        const shouldRetryTimeouts = options?.doNotRetryTimeouts !== true;
+        let lastError = null;
+        for (let attempt = 0; attempt <= BaseApiClient.maxRetries; attempt++) {
+            // Wait for rate limiter slot before making request
+            await BaseApiClient.rateLimiter.waitForSlot();
+            // Create AbortController for timeout
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+            try {
+                const response = await fetch(url, {
+                    method,
+                    headers,
+                    body: body !== undefined ? (typeof body === 'string' ? body : JSON.stringify(body)) : undefined,
+                    signal: controller.signal,
+                });
+                // Update rate limiter state from response headers
+                BaseApiClient.rateLimiter.updateFromHeaders(response.headers);
+                // Handle rate limiting (429)
+                if (response.status === 429) {
+                    if (attempt < BaseApiClient.maxRetries) {
+                        await BaseApiClient.rateLimiter.handleRateLimited(response);
+                        continue; // Retry
+                    }
+                    // Max retries exceeded
+                    const error = await response.json().catch(() => ({ error: { message: 'Rate limit exceeded' } }));
+                    throw new Error(error.error?.message || 'Rate limit exceeded');
+                }
+                if (!response.ok) {
+                    const error = await response.json().catch(() => ({ error: { message: response.statusText } }));
+                    throw new ApiStorageClientError(error.error?.message || `API request failed: ${response.status}`, response.status);
+                }
+                // Handle empty responses (204 No Content)
+                if (response.status === 204) {
+                    return undefined;
+                }
+                // Handle text/plain responses
+                const responseContentType = response.headers.get('content-type') || '';
+                if (responseContentType.includes('text/plain')) {
+                    return response.text();
+                }
+                // Handle binary responses
+                if (responseContentType.includes('application/octet-stream')) {
+                    const buffer = Buffer.from(await response.arrayBuffer());
+                    return buffer;
+                }
+                const json = await response.json();
+                return unwrapData(json);
+            }
+            catch (err) {
+                // Handle timeout/abort errors
+                if (err.name === 'AbortError') {
+                    const timeoutError = new Error(`Request timed out after ${timeoutMs}ms`);
+                    timeoutError.isTimeout = true;
+                    lastError = timeoutError;
+                    // Don't retry if doNotRetryTimeouts is true
+                    if (!shouldRetryTimeouts || attempt >= BaseApiClient.maxRetries) {
+                        throw timeoutError;
+                    }
+                    // Wait a bit before retrying
+                    await new Promise(resolve => setTimeout(resolve, 100 * (attempt + 1)));
+                    continue;
+                }
+                lastError = err;
+                // Don't retry non-timeout errors unless it's a network issue
+                const isNetworkError = !err.response && !err.status;
+                if (!isNetworkError || attempt >= BaseApiClient.maxRetries) {
+                    throw err;
+                }
+                // Wait before retrying network errors
+                await new Promise(resolve => setTimeout(resolve, 100 * (attempt + 1)));
+            }
+            finally {
+                clearTimeout(timeoutId);
+            }
+        }
+        throw lastError || new Error('Request failed after retries');
+    }
+    async requestWithHeaders(method, path, body, contentType) {
+        const url = `${this.config.apiUrl}/v2${path}`;
+        const headers = { ...this.headers };
+        if (contentType) {
+            headers['Content-Type'] = contentType;
+        }
+        for (let attempt = 0; attempt <= BaseApiClient.maxRetries; attempt++) {
+            await BaseApiClient.rateLimiter.waitForSlot();
+            const response = await fetch(url, {
+                method,
+                headers,
+                body: body !== undefined ? (typeof body === 'string' ? body : JSON.stringify(body)) : undefined,
+            });
+            BaseApiClient.rateLimiter.updateFromHeaders(response.headers);
+            if (response.status === 429) {
+                if (attempt < BaseApiClient.maxRetries) {
+                    await BaseApiClient.rateLimiter.handleRateLimited(response);
+                    continue;
+                }
+                const error = await response.json().catch(() => ({ error: { message: 'Rate limit exceeded' } }));
+                throw new Error(error.error?.message || 'Rate limit exceeded');
+            }
+            if (!response.ok) {
+                const error = await response.json().catch(() => ({ error: { message: response.statusText } }));
+                throw new Error(error.error?.message || `API request failed: ${response.status}`);
+            }
+            if (response.status === 204) {
+                return { data: undefined, contentType: '' };
+            }
+            const responseContentType = response.headers.get('content-type') || 'application/json';
+            if (responseContentType.includes('text/plain')) {
+                return { data: await response.text(), contentType: 'text/plain' };
+            }
+            if (responseContentType.includes('application/octet-stream')) {
+                const buffer = Buffer.from(await response.arrayBuffer());
+                return { data: buffer, contentType: 'application/octet-stream' };
+            }
+            const json = await response.json();
+            return { data: json, contentType: responseContentType.split(';')[0].trim() || 'application/json' };
+        }
+        throw new Error('Request failed after retries');
+    }
+}
+/**
+ * Dataset client that communicates via internal API
+ */
+class ApiDatasetClient extends BaseApiClient {
+    id;
+    constructor(config, datasetId) {
+        super(config);
+        this.id = datasetId;
+    }
+    async get() {
+        if (!isStorageUuid(this.id))
+            return undefined;
+        try {
+            const data = await this.request('GET', `/datasets/${this.id}`);
+            return {
+                id: String(data.id ?? this.id),
+                name: data.name ?? undefined,
+                createdAt: parseApiDate(data.createdAt),
+                modifiedAt: parseApiDate(data.modifiedAt),
+                accessedAt: parseApiDate(data.accessedAt),
+                itemCount: typeof data.itemCount === 'number'
+                    ? data.itemCount
+                    : parseInt(String(data.itemCount ?? '0'), 10),
+            };
+        }
+        catch (err) {
+            return handleStorageGetError(err);
+        }
+    }
+    async update(newFields) {
+        throw new Error('Dataset rename not supported from container');
+    }
+    async delete() {
+        await this.request('DELETE', `/datasets/${this.id}`);
+    }
+    async downloadItems() {
+        const result = await this._listItems();
+        return Buffer.from(JSON.stringify(result.items));
+    }
+    listItems(options) {
+        return makeAsyncIterableResult(() => this._listItems(options));
+    }
+    /**
+     * Iterates over dataset entries (index-value pairs) using an async generator.
+     * Required by Crawlee's Dataset.entries() method.
+     */
+    listEntries(options) {
+        return makeAsyncIterableResult(async () => {
+            const result = await this._listItems(options);
+            const entries = result.items.map((item, idx) => [
+                (result.offset ?? 0) + idx,
+                item,
+            ]);
+            return {
+                total: result.total,
+                count: entries.length,
+                offset: result.offset,
+                limit: result.limit,
+                items: entries,
+            };
+        });
+    }
+    async _listItems(options) {
+        const params = new URLSearchParams();
+        const offset = options?.offset ?? 0;
+        const limit = options?.limit;
+        if (options?.offset)
+            params.set('offset', String(options.offset));
+        if (options?.limit)
+            params.set('limit', String(options.limit));
+        if (options?.desc)
+            params.set('desc', 'true');
+        if (options?.fields)
+            params.set('fields', options.fields.join(','));
+        if (options?.clean)
+            params.set('clean', 'true');
+        if (options?.skipHidden)
+            params.set('skipHidden', 'true');
+        if (options?.skipEmpty)
+            params.set('skipEmpty', 'true');
+        const queryString = params.toString();
+        const path = `/datasets/${this.id}/items${queryString ? `?${queryString}` : ''}`;
+        const result = await this.request('GET', path);
+        // v2 returns a bare JSON array; internal API returned { total, count, offset, limit, items }
+        if (Array.isArray(result)) {
+            return {
+                total: result.length,
+                count: result.length,
+                offset,
+                limit: limit ?? result.length,
+                items: result,
+            };
+        }
+        return result;
+    }
+    async pushItems(items) {
+        let parsed;
+        if (typeof items === 'string') {
+            parsed = JSON.parse(items);
+            if (!Array.isArray(parsed))
+                parsed = [parsed];
+        }
+        else if (Array.isArray(items)) {
+            parsed = items.map(i => (typeof i === 'string' ? JSON.parse(i) : i));
+        }
+        else {
+            parsed = [items];
+        }
+        await this.request('POST', `/datasets/${this.id}/items`, parsed);
+    }
+}
+exports.ApiDatasetClient = ApiDatasetClient;
+/**
+ * Key-Value Store client that communicates via internal API
+ */
+class ApiKeyValueStoreClient extends BaseApiClient {
+    id;
+    constructor(config, storeId) {
+        super(config);
+        this.id = storeId;
+    }
+    async get() {
+        if (!isStorageUuid(this.id))
+            return undefined;
+        try {
+            const data = await this.request('GET', `/key-value-stores/${this.id}`);
+            return {
+                id: String(data.id ?? this.id),
+                name: data.name ?? undefined,
+                createdAt: parseApiDate(data.createdAt),
+                modifiedAt: parseApiDate(data.modifiedAt),
+                accessedAt: parseApiDate(data.accessedAt),
+            };
+        }
+        catch (err) {
+            return handleStorageGetError(err);
+        }
+    }
+    async update(newFields) {
+        throw new Error('KVS rename not supported from container');
+    }
+    async delete() {
+        if (!isStorageUuid(this.id))
+            return;
+        await this.request('DELETE', `/key-value-stores/${this.id}`);
+    }
+    async recordExists(key) {
+        try {
+            await this.getRecord(key);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async getRecord(key, options) {
+        try {
+            const { data, contentType } = await this.requestWithHeaders('GET', `/key-value-stores/${this.id}/records/${key}`);
+            if (data === undefined || data === null)
+                return undefined;
+            let value = data;
+            if (options?.buffer && !Buffer.isBuffer(value)) {
+                value = Buffer.from(typeof value === 'string' ? value : JSON.stringify(value));
+            }
+            return { key, value, contentType: contentType || 'application/json' };
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async setRecord(record, options) {
+        const { key, value, contentType } = record;
+        let body;
+        let ct;
+        // Handle Readable streams (e.g., fs.createReadStream) - Apify compatibility
+        // Streams are objects with a pipe method. We need to buffer them.
+        if (value && typeof value === 'object' && typeof value.pipe === 'function') {
+            const chunks = [];
+            for await (const chunk of value) {
+                chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+            }
+            const buffered = Buffer.concat(chunks);
+            body = buffered.toString('base64');
+            ct = 'application/octet-stream';
+        }
+        else if (Buffer.isBuffer(value)) {
+            body = value.toString('base64');
+            ct = 'application/octet-stream';
+        }
+        else if (typeof value === 'string') {
+            body = value;
+            ct = 'text/plain';
+        }
+        else {
+            body = value;
+            ct = 'application/json';
+        }
+        // Pass timeout/retry options from KeyValueStoreRecordOptions to the API request
+        const requestOptions = options ? {
+            timeoutSecs: options.timeoutSecs,
+            doNotRetryTimeouts: options.doNotRetryTimeouts,
+        } : undefined;
+        await this.request('PUT', `/key-value-stores/${this.id}/records/${key}`, body, contentType || ct, requestOptions);
+    }
+    async deleteRecord(key) {
+        await this.request('DELETE', `/key-value-stores/${this.id}/records/${key}`);
+    }
+    async listKeys(options) {
+        const params = new URLSearchParams();
+        if (options?.prefix)
+            params.set('prefix', options.prefix);
+        if (options?.limit)
+            params.set('limit', String(options.limit));
+        if (options?.exclusiveStartKey)
+            params.set('exclusiveStartKey', options.exclusiveStartKey);
+        if (options?.collection)
+            params.set('collection', options.collection);
+        const queryString = params.toString();
+        const path = `/key-value-stores/${this.id}/keys${queryString ? `?${queryString}` : ''}`;
+        return this.request('GET', path);
+    }
+    keys(options) {
+        return (0, kvs_iteration_js_1.makePaginatingKeysIterable)((opts) => this.listKeys(opts), options);
+    }
+    values(options) {
+        return (0, kvs_iteration_js_1.makePaginatingValuesIterable)((opts) => this.listKeys(opts), (key) => this.getRecord(key), options);
+    }
+    entries(options) {
+        return (0, kvs_iteration_js_1.makePaginatingEntriesIterable)((opts) => this.listKeys(opts), (key) => this.getRecord(key), options);
+    }
+    /**
+     * Iterates over all keys in the key-value store, calling the iteratee function for each key.
+     * Similar to Apify's KeyValueStore.forEachKey() method.
+     */
+    async forEachKey(iteratee, options) {
+        return (0, kvs_iteration_js_1.forEachKeyPaginated)((opts) => this.listKeys(opts), iteratee, options);
+    }
+    /**
+     * Gets a value from the key-value store that is automatically saved when modified.
+     * If the key doesn't exist, returns the defaultValue and stores it for future use.
+     * Similar to Apify's KeyValueStore.getAutoSavedValue() method.
+     *
+     * @param key - The key of the record to get
+     * @param defaultValue - The default value to return and store if the key doesn't exist
+     * @returns The stored value or the defaultValue
+     */
+    async getAutoSavedValue(key, defaultValue) {
+        const record = await this.getRecord(key);
+        if (record === undefined) {
+            // Store the default value and return it
+            await this.setRecord({ key, value: defaultValue });
+            return defaultValue;
+        }
+        return record.value;
+    }
+    /**
+     * Returns a URL for the given key that may be used to publicly access the value
+     * in the remote key-value store.
+     * Similar to Apify's KeyValueStore.getPublicUrl() method.
+     *
+     * @param key - The key of the record to get the public URL for
+     * @returns A public URL for accessing the record
+     */
+    getPublicUrl(key) {
+        return `${this.config.apiUrl}/v2/key-value-stores/${this.id}/records/${key}`;
+    }
+}
+exports.ApiKeyValueStoreClient = ApiKeyValueStoreClient;
+/**
+ * Request Queue client that communicates via internal API
+ */
+class ApiRequestQueueClient extends BaseApiClient {
+    id;
+    clientKey;
+    constructor(config, queueId, options) {
+        super(config);
+        this.id = queueId;
+        this.clientKey = options?.clientKey || (0, node_crypto_1.randomUUID)().replace(/-/g, '').slice(0, 16);
+    }
+    async get() {
+        if (!isStorageUuid(this.id))
+            return undefined;
+        try {
+            const result = await this.request('GET', `/request-queues/${this.id}`);
+            return mapRequestQueueInfo(this.id, result);
+        }
+        catch (err) {
+            return handleStorageGetError(err);
+        }
+    }
+    async update(newFields) {
+        if (!isStorageUuid(this.id))
+            return undefined;
+        const result = await this.request('PUT', `/request-queues/${this.id}`, newFields);
+        return { name: result.name ?? newFields.name };
+    }
+    async delete() {
+        if (!isStorageUuid(this.id))
+            return;
+        await this.request('DELETE', `/request-queues/${this.id}`);
+    }
+    async addRequest(request, options) {
+        const query = buildRqQueryParams(this.clientKey, { forefront: options?.forefront });
+        return this.request('POST', `/request-queues/${this.id}/requests${query}`, request);
+    }
+    async batchAddRequests(requests, options) {
+        const processedRequests = [];
+        const unprocessedRequests = [];
+        const BATCH_SIZE = 25;
+        for (let i = 0; i < requests.length; i += BATCH_SIZE) {
+            const chunk = requests.slice(i, i + BATCH_SIZE);
+            try {
+                const query = buildRqQueryParams(this.clientKey, { forefront: options?.forefront });
+                const result = await this.request('POST', `/request-queues/${this.id}/requests/batch${query}`, chunk);
+                processedRequests.push(...result.processedRequests);
+                unprocessedRequests.push(...result.unprocessedRequests);
+            }
+            catch {
+                for (const request of chunk) {
+                    unprocessedRequests.push({
+                        uniqueKey: request.uniqueKey,
+                        url: request.url,
+                        method: request.method,
+                    });
+                }
+            }
+        }
+        return { processedRequests, unprocessedRequests };
+    }
+    async getRequest(id) {
+        try {
+            return await this.request('GET', `/request-queues/${this.id}/requests/${id}`);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async updateRequest(request, options) {
+        const query = buildRqQueryParams(this.clientKey, { forefront: options?.forefront });
+        return this.request('PUT', `/request-queues/${this.id}/requests/${request.id}${query}`, request);
+    }
+    async deleteRequest(id) {
+        await this.request('DELETE', `/request-queues/${this.id}/requests/${id}`);
+    }
+    async listHead(options) {
+        const query = buildRqQueryParams(this.clientKey, { limit: options?.limit });
+        const result = await this.request('GET', `/request-queues/${this.id}/head${query}`);
+        return {
+            limit: result.limit,
+            queueModifiedAt: result.queueModifiedAt ? new Date(result.queueModifiedAt) : new Date(),
+            hadMultipleClients: result.hadMultipleClients,
+            items: result.items,
+        };
+    }
+    async listAndLockHead(options) {
+        const query = buildRqQueryParams(this.clientKey, {
+            limit: options.limit || 25,
+            lockSecs: options.lockSecs,
+        });
+        const result = await this.request('POST', `/request-queues/${this.id}/head/lock${query}`);
+        return {
+            ...result,
+            queueModifiedAt: result.queueModifiedAt ? new Date(result.queueModifiedAt) : new Date(),
+        };
+    }
+    async prolongRequestLock(id, options) {
+        const query = buildRqQueryParams(this.clientKey, {
+            lockSecs: options.lockSecs,
+            forefront: options.forefront,
+        });
+        const result = await this.request('PUT', `/request-queues/${this.id}/requests/${id}/lock${query}`);
+        return { lockExpiresAt: new Date(result.lockExpiresAt) };
+    }
+    async deleteRequestLock(id, options) {
+        const query = buildRqQueryParams(this.clientKey, { forefront: options?.forefront });
+        await this.request('DELETE', `/request-queues/${this.id}/requests/${id}/lock${query}`);
+    }
+    /**
+     * List requests in the queue with optional filtering and pagination (Apify REST API).
+     */
+    async listRequests(options) {
+        const params = new URLSearchParams();
+        params.set('clientKey', options?.clientKey ?? this.clientKey);
+        if (options?.limit !== undefined)
+            params.set('limit', String(options.limit));
+        if (options?.cursor)
+            params.set('cursor', options.cursor);
+        if (options?.exclusiveStartId)
+            params.set('exclusiveStartId', options.exclusiveStartId);
+        if (options?.filter?.length) {
+            for (const f of options.filter)
+                params.append('filter', f);
+        }
+        const query = params.toString();
+        return this.request('GET', `/request-queues/${this.id}/requests${query ? `?${query}` : ''}`);
+    }
+    /**
+     * Unlock all requests locked by this clientKey (Apify REST API).
+     */
+    async unlockRequests(options) {
+        const query = buildRqQueryParams(options.clientKey ?? this.clientKey, {
+            forefront: options.forefront,
+            runId: options.runId,
+        });
+        return this.request('POST', `/request-queues/${this.id}/requests/unlock${query}`);
+    }
+}
+exports.ApiRequestQueueClient = ApiRequestQueueClient;
+/**
+ * Dataset collection client (API-based)
+ */
+class ApiDatasetCollectionClient {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async getOrCreate(name) {
+        const data = await postStorageCollection(this.config, '/datasets', name);
+        return {
+            id: String(data.id),
+            name: data.name ?? name,
+            createdAt: parseApiDate(data.createdAt),
+            modifiedAt: parseApiDate(data.modifiedAt),
+            accessedAt: parseApiDate(data.accessedAt),
+            itemCount: typeof data.itemCount === 'number'
+                ? data.itemCount
+                : parseInt(String(data.itemCount ?? '0'), 10),
+        };
+    }
+    async list() {
+        // Containers don't list all datasets - return empty
+        return { total: 0, count: 0, offset: 0, limit: 0, items: [] };
+    }
+}
+/**
+ * Key-value store collection client (API-based)
+ */
+class ApiKeyValueStoreCollectionClient {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async getOrCreate(name) {
+        const data = await postStorageCollection(this.config, '/key-value-stores', name);
+        return {
+            id: String(data.id),
+            name: data.name ?? name,
+            createdAt: parseApiDate(data.createdAt),
+            modifiedAt: parseApiDate(data.modifiedAt),
+            accessedAt: parseApiDate(data.accessedAt),
+        };
+    }
+    async list() {
+        return { total: 0, count: 0, offset: 0, limit: 0, items: [] };
+    }
+}
+/**
+ * Request queue collection client (API-based)
+ */
+class ApiRequestQueueCollectionClient {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async getOrCreate(name) {
+        const data = await postStorageCollection(this.config, '/request-queues', name);
+        return {
+            id: String(data.id),
+            name: data.name ?? name,
+            createdAt: parseApiDate(data.createdAt),
+            modifiedAt: parseApiDate(data.modifiedAt),
+            accessedAt: parseApiDate(data.accessedAt),
+            totalRequestCount: typeof data.totalRequestCount === 'number'
+                ? data.totalRequestCount
+                : parseInt(String(data.totalRequestCount ?? '0'), 10),
+            handledRequestCount: typeof data.handledRequestCount === 'number'
+                ? data.handledRequestCount
+                : parseInt(String(data.handledRequestCount ?? '0'), 10),
+            pendingRequestCount: typeof data.pendingRequestCount === 'number'
+                ? data.pendingRequestCount
+                : parseInt(String(data.pendingRequestCount ?? '0'), 10),
+            hadMultipleClients: data.hadMultipleClients === true || data.hadMultipleClients === '1',
+        };
+    }
+    async list() {
+        const url = `${this.config.apiUrl}/v2/request-queues`;
+        const response = await fetch(url, {
+            headers: storageCollectionHeaders(this.config),
+        });
+        if (!response.ok) {
+            const error = await response.json().catch(() => ({ error: { message: response.statusText } }));
+            throw new Error(error.error?.message
+                || `Failed to list request queues: ${response.status}`);
+        }
+        const body = await response.json();
+        const data = unwrapData(body);
+        return {
+            total: data.total,
+            count: data.count,
+            offset: data.offset,
+            limit: data.limit,
+            items: data.items.map((item) => mapRequestQueueInfo(String(item.id), item)),
+        };
+    }
+}
+/**
+ * Factory for creating API-based storage clients.
+ * Implements StorageClient interface for Crawlee compatibility.
+ */
+class ApiStorageClientFactory {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    datasets() {
+        return new ApiDatasetCollectionClient(this.config);
+    }
+    dataset(id) {
+        return new ApiDatasetClient(this.config, id);
+    }
+    keyValueStores() {
+        return new ApiKeyValueStoreCollectionClient(this.config);
+    }
+    keyValueStore(id) {
+        return new ApiKeyValueStoreClient(this.config, id);
+    }
+    requestQueues() {
+        return new ApiRequestQueueCollectionClient(this.config);
+    }
+    requestQueue(id, options) {
+        return new ApiRequestQueueClient(this.config, id, options);
+    }
+    async purge() {
+        // Containers don't purge storage
+    }
+    async teardown() {
+        // no-op: cleanup handled by server
+    }
+    async setStatusMessage(message) {
+        console.log(`[Scrapely] Status: ${message}`);
+    }
+}
+exports.ApiStorageClientFactory = ApiStorageClientFactory;