scrapely 1.2.39 → 1.2.41

package/dist/storage/request-queue-client.d.ts

@@ -20,6 +20,15 @@ export interface ListRequestsResult {
 import type { DragonflyClient } from './dragonfly-client.js';
 import type { RequestQueueOptions } from '@crawlee/types';
 import { type RequestQueueRequest } from './rq-list-requests.js';
+/** Longest lock, in seconds (same as the server's MAX_LOCK_SECS). */
+export declare const MAX_LOCK_SECS = 86400;
+/** lockSecs as a whole number of seconds from 1 to MAX_LOCK_SECS. */
+export declare function clampLockSecs(lockSecs: number): number;
+/**
+ * Largest stored request a head read loads (the server's MAX_READABLE_STORED_REQUEST_BYTES):
+ * a larger one is left out of the head instead of being read into memory.
+ */
+export declare const MAX_HEAD_REQUEST_BYTES: number;
 export declare class DragonflyRequestQueueClient implements RequestQueueClient {
     private client;
     private id;
@@ -71,6 +80,8 @@ export declare class DragonflyRequestQueueClient implements RequestQueueClient {
      * @param options.clientKey - Client identifier for hadMultipleClients tracking
      */
     listRequests(options?: ListRequestsOptions): Promise<ListRequestsResult>;
+    /** The ids whose stored request is at most MAX_HEAD_REQUEST_BYTES (HSTRLEN, nothing loaded). */
+    private withinHeadSize;
     listAndLockHead(options: ListAndLockOptions & {
         clientKey?: string;
     }): Promise<ListAndLockHeadResult>;

package/dist/storage/request-queue-client.js

@@ -1,11 +1,34 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DragonflyRequestQueueClient = void 0;
+exports.DragonflyRequestQueueClient = exports.MAX_HEAD_REQUEST_BYTES = exports.MAX_LOCK_SECS = void 0;
+exports.clampLockSecs = clampLockSecs;
 const utils_js_1 = require("./utils.js");
 const billing_tracker_js_1 = require("./billing-tracker.js");
 const rq_lock_script_js_1 = require("./rq-lock-script.js");
 const rq_list_requests_js_1 = require("./rq-list-requests.js");
+const ttl_manager_js_1 = require("./ttl-manager.js");
 const node_crypto_1 = require("node:crypto");
+/**
+ * Lock owners are stored as "clientKey:runId", so a clientKey holding ':'
+ * could present another run's owner value. Same rule as the server.
+ */
+function assertClientKey(clientKey) {
+    if (clientKey.includes(':'))
+        throw new Error("clientKey must not contain ':'");
+    return clientKey;
+}
+/** Longest lock, in seconds (same as the server's MAX_LOCK_SECS). */
+exports.MAX_LOCK_SECS = 86_400;
+/** lockSecs as a whole number of seconds from 1 to MAX_LOCK_SECS. */
+function clampLockSecs(lockSecs) {
+    const n = Math.floor(Number(lockSecs));
+    return Number.isFinite(n) ? Math.min(Math.max(n, 1), exports.MAX_LOCK_SECS) : 1;
+}
+/**
+ * Largest stored request a head read loads (the server's MAX_READABLE_STORED_REQUEST_BYTES):
+ * a larger one is left out of the head instead of being read into memory.
+ */
+exports.MAX_HEAD_REQUEST_BYTES = 10 * 1024 * 1024 + 64 * 1024;
 class DragonflyRequestQueueClient {
     client;
     id;
@@ -18,7 +41,7 @@ class DragonflyRequestQueueClient {
     constructor(client, id, options) {
         this.client = client;
         this.id = id;
-        this.clientKey = options?.clientKey || (0, node_crypto_1.randomUUID)().replace(/-/g, '').slice(0, 16);
+        this.clientKey = assertClientKey(options?.clientKey || (0, node_crypto_1.randomUUID)().replace(/-/g, '').slice(0, 16));
         this.infoKey = client.key('rq', id, 'info');
         this.requestsKey = client.key('rq', id, 'requests');
         this.pendingKey = client.key('rq', id, 'pending');
@@ -119,11 +142,13 @@ class DragonflyRequestQueueClient {
             const raw = await this.client.redis.hget(indexKey, `id:${this.id}`);
             if (raw) {
                 const data = JSON.parse(raw);
-                if (data.name)
-                    await this.client.redis.hdel(indexKey, `name:${data.name}`);
+                // Name pointers are scoped by owner (name:{userId}:{name}).
+                const ownerId = data.userId ?? this.client.billingContext?.userId;
+                await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, ownerId, data.name, this.id);
                 data.name = newFields.name;
                 await this.client.redis.hset(indexKey, `id:${this.id}`, JSON.stringify(data));
-                await this.client.redis.hset(indexKey, `name:${newFields.name}`, JSON.stringify(data));
+                if (ownerId && newFields.name)
+                    await this.client.redis.hset(indexKey, (0, utils_js_1.ownerNameField)(ownerId, newFields.name), JSON.stringify(data));
             }
         }
         await this.client.redis.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
@@ -134,8 +159,7 @@ class DragonflyRequestQueueClient {
         await this.client.redis.del(this.infoKey, this.requestsKey, this.pendingKey, this.lockedKey, this.lockOwnersKey);
         const indexKey = this.client.key('rq:index');
         await this.client.redis.hdel(indexKey, `id:${this.id}`);
-        if (info?.name)
-            await this.client.redis.hdel(indexKey, `name:${info.name}`);
+        await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, info?.userId || this.client.billingContext?.userId, info?.name, this.id);
     }
     async addRequest(request, options) {
         const requestId = request.id || (0, utils_js_1.uniqueKeyToRequestId)(request.uniqueKey);
@@ -156,6 +180,8 @@ class DragonflyRequestQueueClient {
         pipeline.hincrby(this.infoKey, 'totalRequestCount', 1);
         pipeline.hincrby(this.infoKey, 'pendingRequestCount', 1);
         pipeline.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+        // Requests/pending are created by the first add: give them the queue's expiry.
+        (0, ttl_manager_js_1.queueInheritExpiry)(pipeline, this.infoKey, [this.requestsKey, this.pendingKey]);
         await pipeline.exec();
         // Track write and storage bytes for billing
         const bytes = billing_tracker_js_1.BillingTracker.estimateRequestBytes(stored);
@@ -289,7 +315,7 @@ class DragonflyRequestQueueClient {
         const activeLocks = new Set(at(1));
         // Filter out locked items
         const items = [];
-        const candidates = ids.filter((id) => !activeLocks.has(id)).slice(0, limit);
+        const candidates = await this.withinHeadSize(ids.filter((id) => !activeLocks.has(id)).slice(0, limit));
         if (candidates.length > 0) {
             const raws = await this.client.redis.hmget(this.requestsKey, ...candidates);
             for (const raw of raws) {
@@ -408,11 +434,22 @@ class DragonflyRequestQueueClient {
         }
         return result;
     }
+    /** The ids whose stored request is at most MAX_HEAD_REQUEST_BYTES (HSTRLEN, nothing loaded). */
+    async withinHeadSize(ids) {
+        if (ids.length === 0)
+            return ids;
+        const pipeline = this.client.redis.pipeline();
+        for (const id of ids)
+            pipeline.hstrlen(this.requestsKey, id);
+        const sizes = (await pipeline.exec()) ?? [];
+        return ids.filter((_, i) => Number(sizes[i]?.[1] ?? 0) <= exports.MAX_HEAD_REQUEST_BYTES);
+    }
     async listAndLockHead(options) {
         const limit = Math.min(options.limit ?? 25, 25);
         const now = Date.now();
-        const lockUntil = now + options.lockSecs * 1000;
-        const clientKey = options.clientKey || this.clientKey;
+        const lockSecs = clampLockSecs(options.lockSecs);
+        const lockUntil = now + lockSecs * 1000;
+        const clientKey = assertClientKey(options.clientKey || this.clientKey);
         const runId = this.client.billingContext?.runId || null;
         const ownerValue = runId ? `${clientKey}:${runId}` : clientKey;
         const { lockedIds, activeLockCount } = await (0, rq_lock_script_js_1.listAndLockHeadAtomic)(this.client.redis, {
@@ -425,11 +462,19 @@ class DragonflyRequestQueueClient {
             lockUntil,
             limit,
             ownerValue,
+            maxRequestBytes: exports.MAX_HEAD_REQUEST_BYTES,
         });
         const items = [];
         if (lockedIds.length > 0) {
+            // The locked set and owners hash are created by the first lock: give them the queue's expiry.
+            const expiry = this.client.redis.pipeline();
+            (0, ttl_manager_js_1.queueInheritExpiry)(expiry, this.infoKey, [this.lockedKey, this.lockOwnersKey]);
+            await expiry.exec();
+        }
+        const readable = await this.withinHeadSize(lockedIds);
+        if (readable.length > 0) {
             // Single HMGET instead of one HGET round-trip per locked ID
-            const raws = await this.client.redis.hmget(this.requestsKey, ...lockedIds);
+            const raws = await this.client.redis.hmget(this.requestsKey, ...readable);
             for (const raw of raws) {
                 if (!raw)
                     continue;
@@ -453,7 +498,7 @@ class DragonflyRequestQueueClient {
             limit,
             queueModifiedAt: modifiedAt ? new Date(modifiedAt) : (0, utils_js_1.nowISO)(),
             hadMultipleClients,
-            lockSecs: options.lockSecs,
+            lockSecs,
             items,
             // From the atomic lock script: active locks held by ANY client
             // (including this call's own).
@@ -485,7 +530,7 @@ class DragonflyRequestQueueClient {
         if (owner && !(0, rq_lock_script_js_1.matchesLockOwner)(owner, this.clientKey, runId)) {
             throw new Error(`Request with ID ${id} is not locked by this client in queue ${this.id}`);
         }
-        const lockUntil = now + options.lockSecs * 1000;
+        const lockUntil = now + clampLockSecs(options.lockSecs) * 1000;
         await this.client.redis.zadd(this.lockedKey, String(lockUntil), id);
         return { lockExpiresAt: new Date(lockUntil) };
     }
@@ -512,7 +557,7 @@ class DragonflyRequestQueueClient {
             lockOwnersKey: this.lockOwnersKey,
             pendingKey: this.pendingKey,
         }, {
-            clientKey: options.clientKey,
+            clientKey: assertClientKey(options.clientKey),
             forefront: options.forefront,
             runId: options.runId,
         });

package/dist/storage/rq-collection-client.js

@@ -9,18 +9,15 @@ class DragonflyRequestQueueCollectionClient {
     }
     async getOrCreate(name) {
         const indexKey = this.client.key('rq:index');
+        // Every storage belongs to the run's user; the name pointer is scoped by that user.
+        const userId = (0, utils_js_1.requireStorageOwner)(this.client.billingContext, 'request queue');
         if (name) {
-            const existing = await this.client.redis.hget(indexKey, `name:${name}`);
+            const existing = await this.client.redis.hget(indexKey, (0, utils_js_1.ownerNameField)(userId, name));
             if (existing)
                 return JSON.parse(existing);
         }
         const id = (0, utils_js_1.generateId)();
         const now = (0, utils_js_1.nowISO)();
-        // Real userId when billing context is active; legacy placeholder otherwise.
-        // Matches the KVS collection client convention — the server's storage usage
-        // and TTL jobs match on this field to attribute storages to users, so an
-        // empty value would orphan the queue from per-user accounting.
-        const userId = this.client.billingContext?.userId ?? '1';
         const data = {
             id,
             name,
@@ -36,7 +33,7 @@ class DragonflyRequestQueueCollectionClient {
         };
         await this.client.redis.hset(indexKey, `id:${id}`, JSON.stringify(data));
         if (name)
-            await this.client.redis.hset(indexKey, `name:${name}`, JSON.stringify(data));
+            await this.client.redis.hset(indexKey, (0, utils_js_1.ownerNameField)(userId, name), JSON.stringify(data));
         const infoKey = this.client.key('rq', id, 'info');
         const requestsKey = this.client.key('rq', id, 'requests');
         const pendingKey = this.client.key('rq', id, 'pending');
@@ -77,8 +74,7 @@ class DragonflyRequestQueueCollectionClient {
             const exists = await this.client.redis.exists(infoKey);
             if (!exists) {
                 await this.client.redis.hdel(indexKey, `id:${parsed.id}`);
-                if (parsed.name)
-                    await this.client.redis.hdel(indexKey, `name:${parsed.name}`);
+                await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, parsed.userId, parsed.name, parsed.id);
                 continue;
             }
             items.push(parsed);

package/dist/storage/rq-lock-script.d.ts

@@ -9,7 +9,7 @@ import type Redis from 'ioredis';
  * the one calling the endpoint or a different one"). Callers get the flag from
  * this same atomic reply instead of re-reading the whole locked + pending sets.
  */
-export declare const LIST_AND_LOCK_HEAD_LUA = "\nlocal pendingKey = KEYS[1]\nlocal lockedKey = KEYS[2]\nlocal lockOwnersKey = KEYS[3]\nlocal requestsKey = KEYS[4]\nlocal now = tonumber(ARGV[1])\nlocal lockUntil = tonumber(ARGV[2])\nlocal limit = tonumber(ARGV[3])\nlocal ownerValue = ARGV[4]\nlocal scanLimit = tonumber(ARGV[5])\n\nredis.call('ZREMRANGEBYSCORE', lockedKey, '-inf', now)\n\nlocal pendingIds = redis.call('ZRANGE', pendingKey, 0, scanLimit - 1)\nlocal lockedIds = {}\n\nfor _, id in ipairs(pendingIds) do\n  if #lockedIds >= limit then break end\n  local lockScore = redis.call('ZSCORE', lockedKey, id)\n  if not lockScore or tonumber(lockScore) <= now then\n    if redis.call('HEXISTS', requestsKey, id) == 1 then\n      redis.call('ZADD', lockedKey, lockUntil, id)\n      redis.call('HSET', lockOwnersKey, id, ownerValue)\n      table.insert(lockedIds, id)\n    end\n  end\nend\n\n-- Everything left in lockedKey is an active lock (score > now): the purge\n-- above removed the expired ones and we just added ours. An O(1) ZCARD here\n-- replaces the callers' old tail of ZRANGEBYSCORE(locked) + full\n-- ZRANGE(pending, 0, -1) scans on every head fetch.\nreturn { lockedIds, redis.call('ZCARD', lockedKey) }\n";
+export declare const LIST_AND_LOCK_HEAD_LUA = "\nlocal pendingKey = KEYS[1]\nlocal lockedKey = KEYS[2]\nlocal lockOwnersKey = KEYS[3]\nlocal requestsKey = KEYS[4]\nlocal now = tonumber(ARGV[1])\nlocal lockUntil = tonumber(ARGV[2])\nlocal limit = tonumber(ARGV[3])\nlocal ownerValue = ARGV[4]\nlocal scanLimit = tonumber(ARGV[5])\n-- Requests larger than this are never returned by the head, so they are not locked either.\nlocal maxRequestBytes = tonumber(ARGV[6])\n\n-- Expired locks leave lockedKey here, and their owner entries go with them:\n-- otherwise lockOwners keeps one entry per request ever locked.\nlocal expiredIds = redis.call('ZRANGEBYSCORE', lockedKey, '-inf', now)\nfor i = 1, #expiredIds, 1000 do\n  redis.call('HDEL', lockOwnersKey, unpack(expiredIds, i, math.min(i + 999, #expiredIds)))\nend\nredis.call('ZREMRANGEBYSCORE', lockedKey, '-inf', now)\n\nlocal pendingIds = redis.call('ZRANGE', pendingKey, 0, scanLimit - 1)\nlocal lockedIds = {}\n\nfor _, id in ipairs(pendingIds) do\n  if #lockedIds >= limit then break end\n  local lockScore = redis.call('ZSCORE', lockedKey, id)\n  if not lockScore or tonumber(lockScore) <= now then\n    if redis.call('HEXISTS', requestsKey, id) == 1\n      and (not maxRequestBytes or redis.call('HSTRLEN', requestsKey, id) <= maxRequestBytes) then\n      redis.call('ZADD', lockedKey, lockUntil, id)\n      redis.call('HSET', lockOwnersKey, id, ownerValue)\n      table.insert(lockedIds, id)\n    end\n  end\nend\n\n-- Everything left in lockedKey is an active lock (score > now): the purge\n-- above removed the expired ones and we just added ours. An O(1) ZCARD here\n-- replaces the callers' old tail of ZRANGEBYSCORE(locked) + full\n-- ZRANGE(pending, 0, -1) scans on every head fetch.\nreturn { lockedIds, redis.call('ZCARD', lockedKey) }\n";
 export interface ListAndLockHeadKeys {
     pendingKey: string;
     lockedKey: string;
@@ -23,6 +23,8 @@ export interface ListAndLockHeadParams {
     ownerValue: string;
     /** How many pending IDs to scan (should be >= limit). */
     scanLimit?: number;
+    /** Skip (don't lock) requests stored larger than this many bytes. */
+    maxRequestBytes?: number;
 }
 export interface ListAndLockHeadAtomicResult {
     /** IDs locked by this call, in queue-head order. */
@@ -36,14 +38,18 @@ export interface ListAndLockHeadAtomicResult {
 export declare function listAndLockHeadAtomic(redis: Redis, keys: ListAndLockHeadKeys, params: ListAndLockHeadParams): Promise<ListAndLockHeadAtomicResult>;
 /** Returns true if lockOwners entry matches the caller's clientKey (and optional runId). */
 export declare function matchesLockOwner(owner: string, clientKey: string, runId?: string | null): boolean;
+/** Locked requests UNLOCK_REQUESTS_LUA reads per page. */
+export declare const UNLOCK_SCAN_PAGE = 1000;
 /**
- * Atomic unlock-by-owner: scans lockOwners, unlocks every request whose owner
- * matches the clientKey (and optional runId) using the same rules as the old
- * per-request loop, optionally re-queueing unlocked requests at the forefront.
- * Returns the number of unlocked requests. One round-trip, no race window
- * between ZREM and HDEL, and the whole owners hash never leaves the server.
+ * Atomic unlock-by-owner: walks the locked set a page at a time by rank,
+ * unlocks every request whose owner matches the clientKey (and optional
+ * runId) using the same rules as the old per-request loop, optionally
+ * re-queueing unlocked requests at the forefront. Returns the number of
+ * unlocked requests. One round-trip, no race window between ZREM and HDEL,
+ * and neither the owners hash nor the locked set is read whole (pages of
+ * UNLOCK_SCAN_PAGE, ARGV[4]).
  */
-export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n  expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n  local requestId = flat[i]\n  local owner = flat[i + 1]\n  -- In an Actor run, unlock both (1) requests locked by this\n  -- specific run (clientKey:runId) and (2) requests locked by the same\n  -- clientKey without run context (clientKey), but NOT requests locked by the\n  -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n  if owner == expectedOwner or owner == clientKey\n      or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then\n    redis.call('ZREM', lockedKey, requestId)\n    redis.call('HDEL', lockOwnersKey, requestId)\n    if forefront == '1' then\n      redis.call('ZADD', pendingKey, 0, requestId)\n    end\n    unlockedCount = unlockedCount + 1\n  end\nend\n\nreturn unlockedCount\n";
+export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\nlocal pageSize = tonumber(ARGV[4])\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n  expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal unlockedCount = 0\nlocal start = 0\n\nwhile true do\n  local ids = redis.call('ZRANGE', lockedKey, start, start + pageSize - 1)\n  if #ids == 0 then break end\n  local owners = redis.call('HMGET', lockOwnersKey, unpack(ids))\n  local removed = 0\n  for i, requestId in ipairs(ids) do\n    local owner = owners[i]\n    -- In an Actor run, unlock both (1) requests locked by this\n    -- specific run (clientKey:runId) and (2) requests locked by the same\n    -- clientKey without run context (clientKey), but NOT requests locked by the\n    -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n    -- Exact clientKey, or this run's clientKey:runId. An empty runId must not\n    -- unlock every owner that merely starts with \"clientKey:\".\n    if owner and (owner == expectedOwner or owner == clientKey) then\n      redis.call('ZREM', lockedKey, requestId)\n      redis.call('HDEL', lockOwnersKey, requestId)\n      if forefront == '1' then\n        redis.call('ZADD', pendingKey, 0, requestId)\n      end\n      unlockedCount = unlockedCount + 1\n      removed = removed + 1\n    end\n  end\n  if #ids < pageSize then break end\n  -- Removed locks were in this page, so the next unread lock moved up by that many.\n  start = start + #ids - removed\nend\n\nreturn unlockedCount\n";
 export interface UnlockRequestsKeys {
     lockedKey: string;
     lockOwnersKey: string;

package/dist/storage/rq-lock-script.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UNLOCK_REQUESTS_LUA = exports.LIST_AND_LOCK_HEAD_LUA = void 0;
+exports.UNLOCK_REQUESTS_LUA = exports.UNLOCK_SCAN_PAGE = exports.LIST_AND_LOCK_HEAD_LUA = void 0;
 exports.listAndLockHeadAtomic = listAndLockHeadAtomic;
 exports.matchesLockOwner = matchesLockOwner;
 exports.unlockRequestsAtomic = unlockRequestsAtomic;
@@ -24,7 +24,15 @@ local lockUntil = tonumber(ARGV[2])
 local limit = tonumber(ARGV[3])
 local ownerValue = ARGV[4]
 local scanLimit = tonumber(ARGV[5])
+-- Requests larger than this are never returned by the head, so they are not locked either.
+local maxRequestBytes = tonumber(ARGV[6])
 
+-- Expired locks leave lockedKey here, and their owner entries go with them:
+-- otherwise lockOwners keeps one entry per request ever locked.
+local expiredIds = redis.call('ZRANGEBYSCORE', lockedKey, '-inf', now)
+for i = 1, #expiredIds, 1000 do
+  redis.call('HDEL', lockOwnersKey, unpack(expiredIds, i, math.min(i + 999, #expiredIds)))
+end
 redis.call('ZREMRANGEBYSCORE', lockedKey, '-inf', now)
 
 local pendingIds = redis.call('ZRANGE', pendingKey, 0, scanLimit - 1)
@@ -34,7 +42,8 @@ for _, id in ipairs(pendingIds) do
   if #lockedIds >= limit then break end
   local lockScore = redis.call('ZSCORE', lockedKey, id)
   if not lockScore or tonumber(lockScore) <= now then
-    if redis.call('HEXISTS', requestsKey, id) == 1 then
+    if redis.call('HEXISTS', requestsKey, id) == 1
+      and (not maxRequestBytes or redis.call('HSTRLEN', requestsKey, id) <= maxRequestBytes) then
       redis.call('ZADD', lockedKey, lockUntil, id)
       redis.call('HSET', lockOwnersKey, id, ownerValue)
       table.insert(lockedIds, id)
@@ -50,7 +59,7 @@ return { lockedIds, redis.call('ZCARD', lockedKey) }
 `;
 async function listAndLockHeadAtomic(redis, keys, params) {
     const scanLimit = params.scanLimit ?? Math.max(params.limit * 4, 100);
-    const result = await redis.eval(exports.LIST_AND_LOCK_HEAD_LUA, 4, keys.pendingKey, keys.lockedKey, keys.lockOwnersKey, keys.requestsKey, String(params.now), String(params.lockUntil), String(params.limit), params.ownerValue, String(scanLimit));
+    const result = await redis.eval(exports.LIST_AND_LOCK_HEAD_LUA, 4, keys.pendingKey, keys.lockedKey, keys.lockOwnersKey, keys.requestsKey, String(params.now), String(params.lockUntil), String(params.limit), params.ownerValue, String(scanLimit), ...(params.maxRequestBytes !== undefined ? [String(params.maxRequestBytes)] : []));
     // Expected reply shape: [string[], number]. Tolerate degraded replies the
     // same way the old string[] version did (null / JSON-stringified table).
     if (!Array.isArray(result) || result.length < 2) {
@@ -76,12 +85,16 @@ function matchesLockOwner(owner, clientKey, runId) {
     const expectedOwner = runId ? `${clientKey}:${runId}` : clientKey;
     return owner === expectedOwner || owner === clientKey;
 }
+/** Locked requests UNLOCK_REQUESTS_LUA reads per page. */
+exports.UNLOCK_SCAN_PAGE = 1000;
 /**
- * Atomic unlock-by-owner: scans lockOwners, unlocks every request whose owner
- * matches the clientKey (and optional runId) using the same rules as the old
- * per-request loop, optionally re-queueing unlocked requests at the forefront.
- * Returns the number of unlocked requests. One round-trip, no race window
- * between ZREM and HDEL, and the whole owners hash never leaves the server.
+ * Atomic unlock-by-owner: walks the locked set a page at a time by rank,
+ * unlocks every request whose owner matches the clientKey (and optional
+ * runId) using the same rules as the old per-request loop, optionally
+ * re-queueing unlocked requests at the forefront. Returns the number of
+ * unlocked requests. One round-trip, no race window between ZREM and HDEL,
+ * and neither the owners hash nor the locked set is read whole (pages of
+ * UNLOCK_SCAN_PAGE, ARGV[4]).
  */
 exports.UNLOCK_REQUESTS_LUA = `
 local lockedKey = KEYS[1]
@@ -90,38 +103,47 @@ local pendingKey = KEYS[3]
 local clientKey = ARGV[1]
 local runId = ARGV[2]
 local forefront = ARGV[3]
+local pageSize = tonumber(ARGV[4])
 
 local expectedOwner = clientKey
 if runId ~= '' then
   expectedOwner = clientKey .. ':' .. runId
 end
 
-local clientKeyPrefix = clientKey .. ':'
-
-local flat = redis.call('HGETALL', lockOwnersKey)
 local unlockedCount = 0
+local start = 0
 
-for i = 1, #flat, 2 do
-  local requestId = flat[i]
-  local owner = flat[i + 1]
-  -- In an Actor run, unlock both (1) requests locked by this
-  -- specific run (clientKey:runId) and (2) requests locked by the same
-  -- clientKey without run context (clientKey), but NOT requests locked by the
-  -- same clientKey in DIFFERENT runs (clientKey:otherRunId).
-  if owner == expectedOwner or owner == clientKey
-      or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then
-    redis.call('ZREM', lockedKey, requestId)
-    redis.call('HDEL', lockOwnersKey, requestId)
-    if forefront == '1' then
-      redis.call('ZADD', pendingKey, 0, requestId)
+while true do
+  local ids = redis.call('ZRANGE', lockedKey, start, start + pageSize - 1)
+  if #ids == 0 then break end
+  local owners = redis.call('HMGET', lockOwnersKey, unpack(ids))
+  local removed = 0
+  for i, requestId in ipairs(ids) do
+    local owner = owners[i]
+    -- In an Actor run, unlock both (1) requests locked by this
+    -- specific run (clientKey:runId) and (2) requests locked by the same
+    -- clientKey without run context (clientKey), but NOT requests locked by the
+    -- same clientKey in DIFFERENT runs (clientKey:otherRunId).
+    -- Exact clientKey, or this run's clientKey:runId. An empty runId must not
+    -- unlock every owner that merely starts with "clientKey:".
+    if owner and (owner == expectedOwner or owner == clientKey) then
+      redis.call('ZREM', lockedKey, requestId)
+      redis.call('HDEL', lockOwnersKey, requestId)
+      if forefront == '1' then
+        redis.call('ZADD', pendingKey, 0, requestId)
+      end
+      unlockedCount = unlockedCount + 1
+      removed = removed + 1
     end
-    unlockedCount = unlockedCount + 1
   end
+  if #ids < pageSize then break end
+  -- Removed locks were in this page, so the next unread lock moved up by that many.
+  start = start + #ids - removed
 end
 
 return unlockedCount
 `;
 async function unlockRequestsAtomic(redis, keys, params) {
-    const result = await redis.eval(exports.UNLOCK_REQUESTS_LUA, 3, keys.lockedKey, keys.lockOwnersKey, keys.pendingKey, params.clientKey, params.runId ?? '', params.forefront ? '1' : '0');
+    const result = await redis.eval(exports.UNLOCK_REQUESTS_LUA, 3, keys.lockedKey, keys.lockOwnersKey, keys.pendingKey, params.clientKey, params.runId ?? '', params.forefront ? '1' : '0', String(exports.UNLOCK_SCAN_PAGE));
     return typeof result === 'number' ? result : parseInt(String(result), 10) || 0;
 }

package/dist/storage/run-metadata.js

@@ -208,6 +208,12 @@ class RunMetadataManager {
             ttlPipeline.ttl(this.getRunKey(runId));
         }
         const ttlResults = await ttlPipeline.exec();
+        // Runs whose hash is gone (TTL = -2: expired or deleted outside delete()) leave the
+        // sorted set here, so it doesn't keep one entry per run ever created.
+        const staleIds = olderIds.filter((_, i) => ttlResults?.[i]?.[1] === -2);
+        if (staleIds.length > 0) {
+            await this.client.redis.zrem(userRunsKey, ...staleIds);
+        }
         // Collect only the runs that were demoted (exist but have no expiry, TTL = -1)
         const demotedIds = olderIds.filter((_, i) => ttlResults?.[i]?.[1] === -1);
         if (demotedIds.length === 0) {

package/dist/storage/ttl-manager.d.ts

@@ -1,4 +1,7 @@
 import type Redis from 'ioredis';
+import type { ChainableCommander } from 'ioredis';
+/** Queue INHERIT_EXPIRY_LUA: the lazily created `keys` expire with `infoKey` (nothing when it has no expiry). */
+export declare function queueInheritExpiry(pipeline: ChainableCommander, infoKey: string, keys: string[]): void;
 /**
  * TTL configuration in seconds
  */

package/dist/storage/ttl-manager.js

@@ -1,6 +1,26 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TtlManager = exports.DEFAULT_TTL_CONFIG = void 0;
+exports.queueInheritExpiry = queueInheritExpiry;
+/**
+ * KEYS[1]: a storage's info key; KEYS[2..]: its data keys. Gives each data key the info key's
+ * remaining expiry (NX: one that has an expiry keeps it). An expiry set at creation is a no-op
+ * on data keys that don't exist yet, so a key created lazily on the first write never expired.
+ */
+const INHERIT_EXPIRY_LUA = `
+local ttl = redis.call('PTTL', KEYS[1])
+if ttl <= 0 then return 0 end
+for i = 2, #KEYS do
+  redis.call('PEXPIRE', KEYS[i], ttl, 'NX')
+end
+return 1
+`;
+/** Queue INHERIT_EXPIRY_LUA: the lazily created `keys` expire with `infoKey` (nothing when it has no expiry). */
+function queueInheritExpiry(pipeline, infoKey, keys) {
+    if (keys.length === 0)
+        return;
+    pipeline.eval(INHERIT_EXPIRY_LUA, keys.length + 1, infoKey, ...keys);
+}
 /**
  * Default TTL configuration
  */

package/dist/storage/utils.d.ts

@@ -9,3 +9,22 @@ export declare function nowISO(): Date;
  * segments even when they are percent-encoded.
  */
 export declare function pathSegment(value: string): string;
+/**
+ * Field of a named storage in the global `{prefix}datasets:index` /
+ * `kvs:index` / `rq:index` hash: scoped by owner (`name:{userId}:{name}`, the
+ * same field the server writes), so two accounts using one name never share
+ * or overwrite each other's pointer.
+ */
+export declare function ownerNameField(userId: string, name: string): string;
+/** The owner a new storage is created for: the active billing context's user. */
+export declare function requireStorageOwner(billingContext: {
+    userId?: string;
+} | undefined, kind: string): string;
+/**
+ * Remove an owner's name pointer, but only while it still points at `storageId`
+ * (a pointer the owner has since moved to another storage is left alone).
+ */
+export declare function releaseOwnerName(redis: {
+    hget(key: string, field: string): Promise<string | null>;
+    hdel(key: string, ...fields: string[]): Promise<number>;
+}, indexKey: string, userId: string | null | undefined, name: string | null | undefined, storageId: string): Promise<void>;

package/dist/storage/utils.js

@@ -4,6 +4,9 @@ exports.uniqueKeyToRequestId = uniqueKeyToRequestId;
 exports.generateId = generateId;
 exports.nowISO = nowISO;
 exports.pathSegment = pathSegment;
+exports.ownerNameField = ownerNameField;
+exports.requireStorageOwner = requireStorageOwner;
+exports.releaseOwnerName = releaseOwnerName;
 const node_crypto_1 = require("node:crypto");
 function uniqueKeyToRequestId(uniqueKey) {
     return (0, node_crypto_1.createHash)('sha256')
@@ -34,3 +37,39 @@ function pathSegment(value) {
     }
     return encodeURIComponent(value);
 }
+/**
+ * Field of a named storage in the global `{prefix}datasets:index` /
+ * `kvs:index` / `rq:index` hash: scoped by owner (`name:{userId}:{name}`, the
+ * same field the server writes), so two accounts using one name never share
+ * or overwrite each other's pointer.
+ */
+function ownerNameField(userId, name) {
+    return `name:${userId}:${name}`;
+}
+/** The owner a new storage is created for: the active billing context's user. */
+function requireStorageOwner(billingContext, kind) {
+    const userId = billingContext?.userId;
+    if (!userId)
+        throw new Error(`Cannot create a ${kind} without a billing context (no owner user id).`);
+    return userId;
+}
+/**
+ * Remove an owner's name pointer, but only while it still points at `storageId`
+ * (a pointer the owner has since moved to another storage is left alone).
+ */
+async function releaseOwnerName(redis, indexKey, userId, name, storageId) {
+    if (!userId || !name)
+        return;
+    const field = ownerNameField(userId, name);
+    const raw = await redis.hget(indexKey, field);
+    if (!raw)
+        return;
+    try {
+        if (JSON.parse(raw)?.id !== storageId)
+            return;
+    }
+    catch {
+        return;
+    }
+    await redis.hdel(indexKey, field);
+}

package/dist/utils.d.ts

@@ -26,6 +26,8 @@ export interface EnqueueLinksOptions {
         url: string;
         userData?: Record<string, any>;
     } | null;
+    /** Most requests to enqueue from this page. Default: no limit */
+    limit?: number;
 }
 /**
  * Enqueues links found on the page using a CSS selector.

package/dist/utils.js

@@ -4,6 +4,8 @@ exports.utils = void 0;
 exports.enqueueLinks = enqueueLinks;
 exports.requestAsBrowser = requestAsBrowser;
 exports.getRandomUserAgent = getRandomUserAgent;
+/** Requests sent to the queue per addRequests call. */
+const ENQUEUE_BATCH_SIZE = 100;
 /**
  * Enqueues links found on the page using a CSS selector.
  *
@@ -22,7 +24,7 @@ exports.getRandomUserAgent = getRandomUserAgent;
  * ```
  */
 async function enqueueLinks(options) {
-    const { $, requestQueue, selector = 'a[href]', baseUrl, userData = {}, transformRequestFunction } = options;
+    const { $, requestQueue, selector = 'a[href]', baseUrl, userData = {}, transformRequestFunction, limit = Infinity, } = options;
     // Collect all matching URLs
     const urls = [];
     $(selector).each((_, element) => {
@@ -41,8 +43,11 @@ async function enqueueLinks(options) {
     });
     // Deduplicate URLs
     const uniqueUrls = [...new Set(urls)];
-    // Add each URL to the request queue
+    // Build the requests (up to `limit`), then add them to the queue in batches
+    const requests = [];
     for (const url of uniqueUrls) {
+        if (requests.length >= limit)
+            break;
         const request = {
             url,
             userData: { ...userData },
@@ -51,9 +56,11 @@ async function enqueueLinks(options) {
         const transformed = transformRequestFunction
             ? transformRequestFunction(request)
             : request;
-        if (transformed) {
-            await requestQueue.addRequest(transformed);
-        }
+        if (transformed)
+            requests.push(transformed);
+    }
+    for (let i = 0; i < requests.length; i += ENQUEUE_BATCH_SIZE) {
+        await requestQueue.addRequests(requests.slice(i, i + ENQUEUE_BATCH_SIZE));
     }
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.39",
+    "version": "1.2.41",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -44,6 +44,7 @@
         "test:charge-multi-event": "npx tsx --test tests/charge-multi-event.test.ts",
         "test:charge-multi-event-void": "npx tsx --test tests/charge-multi-event-void.test.ts",
         "test:charge-edge-cases": "npx tsx --test tests/charge-edge-cases.test.ts",
+        "test:charge-atomic-refusal-message": "npx tsx --test tests/charge-atomic-refusal-message.test.ts",
         "test:charge-pending-persist": "npx tsx --test tests/charge-pending-persist.test.ts",
         "test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
         "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
@@ -89,20 +90,20 @@
         "crawlee": "^3.16.0",
         "dockerode": "^4.0.9",
         "ioredis": "^6.0.0",
-        "ip-address": "10.3.1",
+        "ip-address": "10.6.0",
         "lodash": "^4.18.1",
         "mysql2": "^3.23.1",
         "playwright": "^1.58.2",
         "scrapely-client": "^2.25.9",
         "tar-fs": "2.1.5",
-        "ws": "^8.18.0"
+        "ws": "^8.21.0"
     },
     "devDependencies": {
         "@types/dockerode": "^4.0.1",
         "@types/node": "^25.5.0",
         "@types/ws": "^8.5.10",
         "dotenv": "^17.3.1",
-        "puppeteer-core": "^24.43.1",
+        "puppeteer-core": "^25.16.0",
         "tsx": "^4.21.0",
         "typescript": "^5.7.0"
     },
@@ -113,8 +114,10 @@
         "axios": "^1.20.0",
         "brace-expansion": "5.0.9",
         "fast-xml-builder": "^1.2.1",
+        "strnum": "^2.4.2",
         "file-type": "^21.3.3",
         "follow-redirects": "^1.16.1",
+        "protobufjs": "^7.6.6",
         "@crawlee/core": {
             "ow": "0.28.2"
         },