scrapely 1.2.3 → 1.2.5

package/dist/storage/request-queue-client.js

@@ -0,0 +1,522 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DragonflyRequestQueueClient = void 0;
+const utils_js_1 = require("./utils.js");
+const billing_tracker_js_1 = require("./billing-tracker.js");
+const rq_lock_script_js_1 = require("./rq-lock-script.js");
+const rq_list_requests_js_1 = require("./rq-list-requests.js");
+const node_crypto_1 = require("node:crypto");
+class DragonflyRequestQueueClient {
+    client;
+    id;
+    infoKey;
+    requestsKey;
+    pendingKey;
+    lockedKey;
+    lockOwnersKey;
+    clientKey;
+    constructor(client, id, options) {
+        this.client = client;
+        this.id = id;
+        this.clientKey = options?.clientKey || (0, node_crypto_1.randomUUID)().replace(/-/g, '').slice(0, 16);
+        this.infoKey = client.key('rq', id, 'info');
+        this.requestsKey = client.key('rq', id, 'requests');
+        this.pendingKey = client.key('rq', id, 'pending');
+        this.lockedKey = client.key('rq', id, 'locked');
+        this.lockOwnersKey = client.key('rq', id, 'lockOwners');
+    }
+    estimateStoredRequestBytes(raw) {
+        return raw ? Buffer.byteLength(raw, 'utf8') : 0;
+    }
+    /**
+     * Track a write operation for billing.
+     */
+    async trackWrite(count = 1) {
+        if (this.client.billingContext) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            await billingTracker.trackWrite(this.client.billingContext.userId, count);
+            await this.client.redis.hincrby(this.infoKey, 'writeCount', count);
+            // Register this queue as accessed for per-run aggregation
+            await this.client.redis.sadd(this.client.key('run', this.client.billingContext.runId, 'rqs_accessed'), this.id);
+        }
+    }
+    /**
+     * Track a read operation for billing.
+     */
+    async trackRead(count = 1, bytesTransferred = 0) {
+        if (this.client.billingContext) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            await billingTracker.trackRead(this.client.billingContext.userId, count);
+            await this.client.redis.hincrby(this.infoKey, 'readCount', count);
+            // Track internal data transfer (bytes returned to actor)
+            if (bytesTransferred > 0) {
+                await billingTracker.trackInternalTransfer(this.client.billingContext.runId, this.client.billingContext.userId, bytesTransferred);
+            }
+            // Register this queue as accessed for per-run aggregation
+            await this.client.redis.sadd(this.client.key('run', this.client.billingContext.runId, 'rqs_accessed'), this.id);
+        }
+    }
+    /**
+     * Track storage bytes for billing.
+     */
+    async trackStorageBytes(bytes) {
+        if (this.client.billingContext) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            await billingTracker.trackStorageBytes(this.client.billingContext.runId, this.client.billingContext.userId, bytes);
+        }
+    }
+    /**
+     * Track clientKey access for hadMultipleClients (per Apify spec).
+     * Returns the current hadMultipleClients flag after update.
+     */
+    async trackClientKey(clientKey) {
+        if (clientKey) {
+            const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
+            if (existingClientKey && existingClientKey !== clientKey) {
+                await this.client.redis.hset(this.infoKey, 'hadMultipleClients', '1');
+            }
+            else if (!existingClientKey) {
+                await this.client.redis.hset(this.infoKey, 'clientKey', clientKey);
+            }
+        }
+        else {
+            const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
+            if (existingClientKey) {
+                await this.client.redis.hset(this.infoKey, 'hadMultipleClients', '1');
+            }
+        }
+        const flag = await this.client.redis.hget(this.infoKey, 'hadMultipleClients');
+        return flag === '1';
+    }
+    async get() {
+        const info = await this.client.redis.hgetall(this.infoKey);
+        if (!info || !info.id)
+            return undefined;
+        return {
+            id: info.id,
+            name: info.name || undefined,
+            userId: info.userId,
+            createdAt: new Date(info.createdAt),
+            modifiedAt: new Date(info.modifiedAt),
+            accessedAt: new Date(info.accessedAt),
+            totalRequestCount: parseInt(info.totalRequestCount || '0', 10),
+            handledRequestCount: parseInt(info.handledRequestCount || '0', 10),
+            pendingRequestCount: parseInt(info.pendingRequestCount || '0', 10),
+            hadMultipleClients: info.hadMultipleClients === '1',
+            stats: {
+                readCount: parseInt(info.readCount || '0', 10),
+                writeCount: parseInt(info.writeCount || '0', 10),
+                deleteCount: parseInt(info.deleteCount || '0', 10),
+                headItemReadCount: parseInt(info.headItemReadCount || '0', 10),
+                storageBytes: parseInt(info.storageBytes || '0', 10),
+            },
+        };
+    }
+    async update(newFields) {
+        if (newFields.name !== undefined) {
+            await this.client.redis.hset(this.infoKey, 'name', newFields.name);
+            const indexKey = this.client.key('rq:index');
+            const raw = await this.client.redis.hget(indexKey, `id:${this.id}`);
+            if (raw) {
+                const data = JSON.parse(raw);
+                if (data.name)
+                    await this.client.redis.hdel(indexKey, `name:${data.name}`);
+                data.name = newFields.name;
+                await this.client.redis.hset(indexKey, `id:${this.id}`, JSON.stringify(data));
+                await this.client.redis.hset(indexKey, `name:${newFields.name}`, JSON.stringify(data));
+            }
+        }
+        await this.client.redis.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+        return { name: newFields.name };
+    }
+    async delete() {
+        const info = await this.client.redis.hgetall(this.infoKey);
+        await this.client.redis.del(this.infoKey, this.requestsKey, this.pendingKey, this.lockedKey, this.lockOwnersKey);
+        const indexKey = this.client.key('rq:index');
+        await this.client.redis.hdel(indexKey, `id:${this.id}`);
+        if (info?.name)
+            await this.client.redis.hdel(indexKey, `name:${info.name}`);
+    }
+    async addRequest(request, options) {
+        const requestId = request.id || (0, utils_js_1.uniqueKeyToRequestId)(request.uniqueKey);
+        const existing = await this.client.redis.hget(this.requestsKey, requestId);
+        if (existing) {
+            const parsed = JSON.parse(existing);
+            return {
+                requestId,
+                wasAlreadyPresent: true,
+                wasAlreadyHandled: !!parsed.handledAt,
+            };
+        }
+        const stored = { ...request, id: requestId };
+        const score = options?.forefront ? 0 : Date.now();
+        const pipeline = this.client.redis.pipeline();
+        pipeline.hset(this.requestsKey, requestId, JSON.stringify(stored));
+        pipeline.zadd(this.pendingKey, String(score), requestId);
+        pipeline.hincrby(this.infoKey, 'totalRequestCount', 1);
+        pipeline.hincrby(this.infoKey, 'pendingRequestCount', 1);
+        pipeline.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+        await pipeline.exec();
+        // Track write and storage bytes for billing
+        const bytes = billing_tracker_js_1.BillingTracker.estimateRequestBytes(stored);
+        await Promise.all([
+            this.trackWrite(1),
+            this.trackStorageBytes(bytes),
+        ]);
+        // Track internal data transfer (bytes written to storage)
+        if (this.client.billingContext) {
+            const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);
+            await billingTracker.trackInternalTransfer(this.client.billingContext.runId, this.client.billingContext.userId, bytes);
+        }
+        return { requestId, wasAlreadyPresent: false, wasAlreadyHandled: false };
+    }
+    async batchAddRequests(requests, options) {
+        const processedRequests = [];
+        const unprocessedRequests = [];
+        for (const request of requests) {
+            try {
+                const result = await this.addRequest(request, options);
+                processedRequests.push({
+                    uniqueKey: request.uniqueKey,
+                    requestId: result.requestId,
+                    wasAlreadyPresent: result.wasAlreadyPresent,
+                    wasAlreadyHandled: result.wasAlreadyHandled,
+                });
+            }
+            catch {
+                unprocessedRequests.push({
+                    uniqueKey: request.uniqueKey,
+                    url: request.url,
+                    method: request.method,
+                });
+            }
+        }
+        return { processedRequests, unprocessedRequests };
+    }
+    async getRequest(id) {
+        const raw = await this.client.redis.hget(this.requestsKey, id);
+        if (!raw)
+            return undefined;
+        // Track read for billing with bytes transferred
+        const bytesTransferred = Buffer.byteLength(raw, 'utf8');
+        await this.trackRead(1, bytesTransferred);
+        return JSON.parse(raw);
+    }
+    async updateRequest(request, options) {
+        const existing = await this.client.redis.hget(this.requestsKey, request.id);
+        if (!existing) {
+            return this.addRequest(request, options);
+        }
+        const old = JSON.parse(existing);
+        const wasHandled = !!old.handledAt;
+        const isNowHandled = !!request.handledAt;
+        const updatedRequest = { ...old, ...request };
+        const oldBytes = this.estimateStoredRequestBytes(existing);
+        const newBytes = billing_tracker_js_1.BillingTracker.estimateRequestBytes(updatedRequest);
+        const deltaBytes = newBytes - oldBytes;
+        await this.client.redis.hset(this.requestsKey, request.id, JSON.stringify(updatedRequest));
+        if (!wasHandled && isNowHandled) {
+            const pipeline = this.client.redis.pipeline();
+            pipeline.zrem(this.pendingKey, request.id);
+            pipeline.zrem(this.lockedKey, request.id);
+            pipeline.hdel(this.lockOwnersKey, request.id);
+            pipeline.hincrby(this.infoKey, 'handledRequestCount', 1);
+            pipeline.hincrby(this.infoKey, 'pendingRequestCount', -1);
+            pipeline.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+            await pipeline.exec();
+        }
+        else if (wasHandled && !isNowHandled) {
+            const score = options?.forefront ? 0 : Date.now();
+            const pipeline = this.client.redis.pipeline();
+            pipeline.zadd(this.pendingKey, String(score), request.id);
+            pipeline.hincrby(this.infoKey, 'handledRequestCount', -1);
+            pipeline.hincrby(this.infoKey, 'pendingRequestCount', 1);
+            pipeline.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+            await pipeline.exec();
+        }
+        if (deltaBytes !== 0) {
+            await this.trackStorageBytes(deltaBytes);
+        }
+        return {
+            requestId: request.id,
+            wasAlreadyPresent: true,
+            wasAlreadyHandled: wasHandled,
+        };
+    }
+    async deleteRequest(id) {
+        const raw = await this.client.redis.hget(this.requestsKey, id);
+        if (!raw)
+            return;
+        const request = JSON.parse(raw);
+        const releasedBytes = this.estimateStoredRequestBytes(raw);
+        const pipeline = this.client.redis.pipeline();
+        pipeline.hdel(this.requestsKey, id);
+        pipeline.zrem(this.pendingKey, id);
+        pipeline.zrem(this.lockedKey, id);
+        pipeline.hdel(this.lockOwnersKey, id);
+        pipeline.hincrby(this.infoKey, 'totalRequestCount', -1);
+        if (request.handledAt) {
+            pipeline.hincrby(this.infoKey, 'handledRequestCount', -1);
+        }
+        else {
+            pipeline.hincrby(this.infoKey, 'pendingRequestCount', -1);
+        }
+        pipeline.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+        await pipeline.exec();
+        if (releasedBytes > 0) {
+            await this.trackStorageBytes(-releasedBytes);
+        }
+    }
+    async listHead(options) {
+        const limit = options?.limit ?? 100;
+        const now = Date.now();
+        // Two round-trips total (same pattern as listRequestsPage): one
+        // pipeline for the pending head + all actively locked IDs (replaces
+        // the per-id ZSCORE loop), then a single HMGET for the request bodies
+        // (replaces the per-id HGET loop).
+        const pipeline = this.client.redis.pipeline();
+        // ioredis 6 types: stop must be string | Buffer (not number)
+        pipeline.zrange(this.pendingKey, 0, String(limit - 1));
+        // The locked ZSET is scored by lockUntil ms, so one exclusive-bound
+        // range read equals the old per-id ZSCORE > now check.
+        pipeline.zrangebyscore(this.lockedKey, `(${now}`, '+inf');
+        const results = await pipeline.exec();
+        const at = (i) => {
+            const value = results?.[i]?.[1];
+            return Array.isArray(value) ? value : [];
+        };
+        const ids = at(0);
+        const activeLocks = new Set(at(1));
+        // Filter out locked items
+        const items = [];
+        const candidates = ids.filter((id) => !activeLocks.has(id)).slice(0, limit);
+        if (candidates.length > 0) {
+            const raws = await this.client.redis.hmget(this.requestsKey, ...candidates);
+            for (const raw of raws) {
+                if (!raw)
+                    continue;
+                const req = JSON.parse(raw);
+                items.push({
+                    id: req.id,
+                    retryCount: req.retryCount ?? 0,
+                    uniqueKey: req.uniqueKey,
+                    url: req.url,
+                    method: req.method ?? 'GET',
+                });
+            }
+        }
+        // Track reads for billing with bytes transferred
+        if (items.length > 0) {
+            const totalBytes = items.reduce((sum, item) => sum + JSON.stringify(item).length, 0);
+            await this.trackRead(items.length, totalBytes);
+        }
+        const modifiedAt = await this.client.redis.hget(this.infoKey, 'modifiedAt');
+        const hadMultipleClients = await this.trackClientKey(this.clientKey);
+        return {
+            limit,
+            queueModifiedAt: modifiedAt ? new Date(modifiedAt) : (0, utils_js_1.nowISO)(),
+            hadMultipleClients,
+            items,
+        };
+    }
+    /**
+     * List all requests in the queue with optional filtering and pagination.
+     * Supports cursor-based pagination (preferred) and deprecated exclusiveStartId.
+     * @param options.limit - Max number of requests to return (default 100, max 10000)
+     * @param options.cursor - Base64-encoded cursor from previous response.
+     *        If the cursor's request was deleted, listing resumes just past its
+     *        sorted position (it never restarts from the first page).
+     * @param options.exclusiveStartId - DEPRECATED: Skip requests up to this ID
+     * @param options.filter - Filter by state: 'locked' or 'pending'
+     * @param options.clientKey - Client identifier for hadMultipleClients tracking
+     */
+    async listRequests(options) {
+        const limit = Math.min(options?.limit ?? 100, 10000);
+        const now = Date.now();
+        // Validate that filter is not used with deprecated exclusiveStartId (per Apify spec)
+        if (options?.filter && options.filter.length > 0 && options?.exclusiveStartId) {
+            throw new Error('filter parameter is not compatible with deprecated exclusiveStartId parameter');
+        }
+        // Track clientKey for hadMultipleClients (per Apify spec)
+        if (options?.clientKey) {
+            const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
+            if (existingClientKey && existingClientKey !== options.clientKey) {
+                await this.client.redis.hset(this.infoKey, 'hadMultipleClients', '1');
+            }
+            else if (!existingClientKey) {
+                await this.client.redis.hset(this.infoKey, 'clientKey', options.clientKey);
+            }
+        }
+        else {
+            // No clientKey provided - treat as new client per Apify spec
+            const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
+            if (existingClientKey) {
+                await this.client.redis.hset(this.infoKey, 'hadMultipleClients', '1');
+            }
+        }
+        // Decode cursor to get starting position
+        let startId;
+        if (options?.cursor) {
+            try {
+                const decoded = JSON.parse(Buffer.from(options.cursor, 'base64').toString('utf8'));
+                startId = decoded.requestId;
+            }
+            catch {
+                // Invalid cursor, start from beginning
+            }
+        }
+        else if (options?.exclusiveStartId) {
+            // Deprecated but still supported
+            startId = options.exclusiveStartId;
+        }
+        // Fetch, filter and paginate via the shared helper. Two round-trips
+        // total: one pipeline (HKEYS + only the ZSET reads the filter needs)
+        // and one HMGET of the page's request bodies.
+        const { items, hasMore } = await (0, rq_list_requests_js_1.listRequestsPage)(this.client.redis, {
+            requestsKey: this.requestsKey,
+            pendingKey: this.pendingKey,
+            lockedKey: this.lockedKey,
+        }, {
+            now,
+            limit,
+            startId,
+            filter: options?.filter,
+        });
+        // Track reads for billing
+        if (items.length > 0) {
+            const totalBytes = items.reduce((sum, item) => sum + JSON.stringify(item).length, 0);
+            await this.trackRead(items.length, totalBytes);
+        }
+        // Build response
+        const result = {
+            items,
+            count: items.length,
+            limit,
+        };
+        // Set exclusiveStartId for backward compatibility
+        if (startId) {
+            result.exclusiveStartId = startId;
+        }
+        // Set cursor in response if one was provided
+        if (options?.cursor) {
+            result.cursor = options.cursor;
+        }
+        // Set nextCursor if there are more results
+        if (hasMore && items.length > 0) {
+            const lastItem = items[items.length - 1];
+            result.nextCursor = Buffer.from(JSON.stringify({ requestId: lastItem.id })).toString('base64');
+        }
+        return result;
+    }
+    async listAndLockHead(options) {
+        const limit = Math.min(options.limit ?? 25, 25);
+        const now = Date.now();
+        const lockUntil = now + options.lockSecs * 1000;
+        const clientKey = options.clientKey || this.clientKey;
+        const runId = this.client.billingContext?.runId || null;
+        const ownerValue = runId ? `${clientKey}:${runId}` : clientKey;
+        const { lockedIds, activeLockCount } = await (0, rq_lock_script_js_1.listAndLockHeadAtomic)(this.client.redis, {
+            pendingKey: this.pendingKey,
+            lockedKey: this.lockedKey,
+            lockOwnersKey: this.lockOwnersKey,
+            requestsKey: this.requestsKey,
+        }, {
+            now,
+            lockUntil,
+            limit,
+            ownerValue,
+        });
+        const items = [];
+        if (lockedIds.length > 0) {
+            // Single HMGET instead of one HGET round-trip per locked ID
+            const raws = await this.client.redis.hmget(this.requestsKey, ...lockedIds);
+            for (const raw of raws) {
+                if (!raw)
+                    continue;
+                const req = JSON.parse(raw);
+                items.push({
+                    id: req.id,
+                    retryCount: req.retryCount ?? 0,
+                    uniqueKey: req.uniqueKey,
+                    url: req.url,
+                    method: req.method ?? 'GET',
+                });
+            }
+        }
+        if (items.length > 0) {
+            const totalBytes = items.reduce((sum, item) => sum + JSON.stringify(item).length, 0);
+            await this.trackRead(items.length, totalBytes);
+        }
+        const modifiedAt = await this.client.redis.hget(this.infoKey, 'modifiedAt');
+        const hadMultipleClients = await this.trackClientKey(clientKey);
+        return {
+            limit,
+            queueModifiedAt: modifiedAt ? new Date(modifiedAt) : (0, utils_js_1.nowISO)(),
+            hadMultipleClients,
+            lockSecs: options.lockSecs,
+            items,
+            // From the atomic lock script: active locks held by ANY client
+            // (including this call's own), per the Apify spec.
+            queueHasLockedRequests: activeLockCount > 0,
+        };
+    }
+    async prolongRequestLock(id, options) {
+        // One pipeline for the three reads instead of sequential round-trips
+        const pipeline = this.client.redis.pipeline();
+        pipeline.hget(this.requestsKey, id);
+        pipeline.zscore(this.lockedKey, id);
+        pipeline.hget(this.lockOwnersKey, id);
+        const results = await pipeline.exec();
+        const raw = results?.[0]?.[1] ?? null;
+        const lockScore = results?.[1]?.[1] ?? null;
+        const owner = results?.[2]?.[1] ?? null;
+        if (!raw) {
+            throw new Error(`Request with ID ${id} not found in queue ${this.id}`);
+        }
+        const request = JSON.parse(raw);
+        if (request.handledAt) {
+            throw new Error(`Request with ID ${id} has already been handled in queue ${this.id}`);
+        }
+        const now = Date.now();
+        if (!lockScore || parseInt(lockScore, 10) <= now) {
+            throw new Error(`Request with ID ${id} is not locked in queue ${this.id}`);
+        }
+        const runId = this.client.billingContext?.runId || null;
+        if (owner && !(0, rq_lock_script_js_1.matchesLockOwner)(owner, this.clientKey, runId)) {
+            throw new Error(`Request with ID ${id} is not locked by this client in queue ${this.id}`);
+        }
+        const lockUntil = now + options.lockSecs * 1000;
+        await this.client.redis.zadd(this.lockedKey, String(lockUntil), id);
+        return { lockExpiresAt: new Date(lockUntil) };
+    }
+    async deleteRequestLock(id, options) {
+        await this.client.redis.zrem(this.lockedKey, id);
+        await this.client.redis.hdel(this.lockOwnersKey, id);
+        if (options?.forefront) {
+            await this.client.redis.zadd(this.pendingKey, '0', id);
+        }
+    }
+    /**
+     * Unlock all requests locked by a specific clientKey.
+     * This is useful for batch unlocking when a client is done processing.
+     * @param options.clientKey - The client key to unlock requests for
+     * @param options.forefront - If true, move unlocked requests to the front of the queue
+     * @param options.runId - Optional run ID to only unlock requests for a specific run
+     */
+    async unlockRequests(options) {
+        // Single atomic round-trip (UNLOCK_REQUESTS_LUA): owner matching, lock
+        // removal and optional forefront re-queueing all happen server-side,
+        // replacing the previous HGETALL + 2-3 round-trips per locked request.
+        const unlockedCount = await (0, rq_lock_script_js_1.unlockRequestsAtomic)(this.client.redis, {
+            lockedKey: this.lockedKey,
+            lockOwnersKey: this.lockOwnersKey,
+            pendingKey: this.pendingKey,
+        }, {
+            clientKey: options.clientKey,
+            forefront: options.forefront,
+            runId: options.runId,
+        });
+        return { unlockedCount };
+    }
+}
+exports.DragonflyRequestQueueClient = DragonflyRequestQueueClient;

package/dist/storage/rq-collection-client.d.ts

@@ -0,0 +1,8 @@
+import type { RequestQueueCollectionClient, RequestQueueInfo, PaginatedList } from '@crawlee/types';
+import type { DragonflyClient } from './dragonfly-client.js';
+export declare class DragonflyRequestQueueCollectionClient implements RequestQueueCollectionClient {
+    private client;
+    constructor(client: DragonflyClient);
+    getOrCreate(name?: string): Promise<RequestQueueInfo>;
+    list(): Promise<PaginatedList<RequestQueueInfo>>;
+}

package/dist/storage/rq-collection-client.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DragonflyRequestQueueCollectionClient = void 0;
+const utils_js_1 = require("./utils.js");
+class DragonflyRequestQueueCollectionClient {
+    client;
+    constructor(client) {
+        this.client = client;
+    }
+    async getOrCreate(name) {
+        const indexKey = this.client.key('rq:index');
+        if (name) {
+            const existing = await this.client.redis.hget(indexKey, `name:${name}`);
+            if (existing)
+                return JSON.parse(existing);
+        }
+        const id = (0, utils_js_1.generateId)();
+        const now = (0, utils_js_1.nowISO)();
+        // Real userId when billing context is active; legacy placeholder otherwise.
+        // Matches the KVS collection client convention — the server's storage usage
+        // and TTL jobs match on this field to attribute storages to users, so an
+        // empty value would orphan the queue from per-user accounting.
+        const userId = this.client.billingContext?.userId ?? '1';
+        const data = {
+            id,
+            name,
+            userId,
+            createdAt: now,
+            modifiedAt: now,
+            accessedAt: now,
+            totalRequestCount: 0,
+            handledRequestCount: 0,
+            pendingRequestCount: 0,
+            hadMultipleClients: false,
+            stats: { readCount: 0, writeCount: 0, deleteCount: 0, headItemReadCount: 0, storageBytes: 0 },
+        };
+        await this.client.redis.hset(indexKey, `id:${id}`, JSON.stringify(data));
+        if (name)
+            await this.client.redis.hset(indexKey, `name:${name}`, JSON.stringify(data));
+        const infoKey = this.client.key('rq', id, 'info');
+        const requestsKey = this.client.key('rq', id, 'requests');
+        const pendingKey = this.client.key('rq', id, 'pending');
+        const lockedKey = this.client.key('rq', id, 'locked');
+        const lockOwnersKey = this.client.key('rq', id, 'lockOwners');
+        await this.client.redis.hset(infoKey, {
+            id,
+            name: name ?? '',
+            userId,
+            totalRequestCount: '0',
+            handledRequestCount: '0',
+            pendingRequestCount: '0',
+            hadMultipleClients: '0',
+            createdAt: now.toISOString(),
+            modifiedAt: now.toISOString(),
+            accessedAt: now.toISOString(),
+        });
+        // Set 24h fallback TTL on all RQ keys (safety net for orphaned RQs from crashed containers)
+        // RQs should be explicitly deleted after run completes, this TTL is just a fallback
+        const ttl = this.client.ttlManager.getTtl('rq', this.client.userPlan);
+        await this.client.ttlManager.setExpiry(this.client.redis, [infoKey, requestsKey, pendingKey, lockedKey, lockOwnersKey], ttl);
+        return data;
+    }
+    async list() {
+        const indexKey = this.client.key('rq:index');
+        const all = await this.client.redis.hgetall(indexKey);
+        const items = [];
+        const seen = new Set();
+        for (const [key, val] of Object.entries(all)) {
+            if (!key.startsWith('id:'))
+                continue;
+            const parsed = JSON.parse(val);
+            if (seen.has(parsed.id))
+                continue;
+            seen.add(parsed.id);
+            const infoKey = this.client.key('rq', parsed.id, 'info');
+            // Skip and clean up stale index entries whose keys have expired via TTL
+            const exists = await this.client.redis.exists(infoKey);
+            if (!exists) {
+                await this.client.redis.hdel(indexKey, `id:${parsed.id}`);
+                if (parsed.name)
+                    await this.client.redis.hdel(indexKey, `name:${parsed.name}`);
+                continue;
+            }
+            items.push(parsed);
+        }
+        return { total: items.length, count: items.length, offset: 0, limit: items.length, items };
+    }
+}
+exports.DragonflyRequestQueueCollectionClient = DragonflyRequestQueueCollectionClient;

package/dist/storage/rq-list-requests.d.ts

@@ -0,0 +1,59 @@
+import type Redis from 'ioredis';
+/** Full request object as returned by listRequests endpoint */
+export interface RequestQueueRequest {
+    id: string;
+    retryCount: number;
+    uniqueKey: string;
+    url: string;
+    method: string;
+    loadedUrl?: string | null;
+    payload?: string | null;
+    noRetry?: boolean;
+    errorMessages?: string[] | null;
+    headers?: Record<string, string> | null;
+    userData?: Record<string, unknown> | null;
+    handledAt?: string | null;
+}
+/** Per-queue Redis keys read by listRequestsPage(). */
+export interface RqListRequestsKeys {
+    requestsKey: string;
+    pendingKey: string;
+    lockedKey: string;
+}
+export interface RqListRequestsParams {
+    now: number;
+    limit: number;
+    /** Cursor / deprecated exclusiveStartId: all IDs <= this one are skipped. */
+    startId?: string;
+    filter?: readonly ('locked' | 'pending')[];
+}
+export interface RqListRequestsPage {
+    items: RequestQueueRequest[];
+    /** True when IDs remain past this page (drives nextCursor). */
+    hasMore: boolean;
+}
+/**
+ * Index of the first element strictly greater than `target` in a
+ * lexicographically sorted array (binary search).
+ *
+ * Unlike indexOf(), it returns the correct position even when `target` itself
+ * is absent, so a cursor item deleted between pages advances pagination past
+ * its old position instead of restarting from page 1 (which used to loop
+ * pagination clients back to the first page forever). This also matches the
+ * documented exclusiveStartId semantics: "all requests up to this one
+ * (including) are skipped from the result".
+ */
+export declare function upperBoundSorted(sortedIds: readonly string[], target: string): number;
+/**
+ * One page of request-queue listing: filtering + cursor pagination in two
+ * round-trips, shared by the storage client and the API server route.
+ *
+ * Round-trip 1 — a single pipeline:
+ *   - HKEYS: all request IDs (field names only, not the bodies);
+ *   - ZRANGEBYSCORE locked (now +inf — only when filtering by 'locked'. The
+ *     locked ZSET is scored by lockUntil ms, so one exclusive-bound range read
+ *     exactly replaces the old per-id ZSCORE loop (score > now);
+ *   - ZRANGE pending 0 -1 — only when filtering by 'pending'.
+ * Round-trip 2 — one HMGET for the page's request bodies.
+ */
+export declare function listRequestsPage(redis: Redis, keys: RqListRequestsKeys, params: RqListRequestsParams): Promise<RqListRequestsPage>;