scrapely 1.2.40 → 1.2.41

package/CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [1.2.41] - 2026-11-08
+
+- An atomic multi-event push (`Actor.pushData(rows, { charges })`) whose charge the limit covers only in part no longer logs "Charging 5 instances of 'audio-minute' event would exceed the end of the account balance - only 3 events were charged". A row is saved only when all of its events are charged, so the SDK gives those 3 back and nothing is charged: the line was untrue. It now says what happened: "Saving this item needs 5 'audio-minute' event(s), but the account balance covers only 3: it was not saved and nothing was charged for it." When some rows still fit, it names how many were saved and charged in full, and when giving a charge back is not confirmed it says so instead of "nothing was charged". A plain `Actor.charge()` that comes back short keeps its line: those events do stay charged.
+- `ChargeResult.coveredCounts` (from `pushData(rows, { charges })`): for each event whose charge came back short, how many events the limit covered (given back, not charged), e.g. `{ 'audio-minute': 3 }`. Actors can cut a row to what the limit covers without reading the log. What is charged and saved does not change.
+- A spending limit of $0 (`maxTotalChargeUsd: 0`) is a real limit, the same as on the platform. It used to be read as "no limit". A missing or non-numeric value still means no limit.
+- A plain `Actor.pushData(items)` to the default dataset of a pay-per-event Actor whose dataset item event is priced goes through the charged push: rows past the spending limit are not saved, and it returns a `ChargeResult`. Before, all rows were saved and only the ones within the limit were paid.
+- A dataset push that fails with a `503 push-outcome-unknown` (the server could not tell whether it was stored) keeps its charge instead of voiding it.
+- `utils.enqueueLinks({ limit })`: enqueue at most `limit` links per page; links are added to the queue in batches of 100.
+- Dragonfly storage clients: named datasets, key-value stores and request queues are looked up per owner (`name:{userId}:{name}`, as the server stores them), a storage is created only with a known owner, and keys created later (dataset items, records, the log truncation marker) take their storage's expiry. Actor runs on the platform use the API storage client and are not affected.
+
 ## [1.2.40] - 2026-10-30
 
 - Actors on puppeteer 25 (ESM-only) are sent to the remote browser when puppeteer is loaded with `require()` before the SDK on a run without the preload. The SDK only recognised puppeteer's old `lib/cjs` build, so such a `puppeteer.launch()` tried to start a local browser; the named `launch` export (`const { launch } = require('puppeteer')`) is covered too. puppeteer 24 and older behave as before.

package/dist/actor.js

@@ -603,6 +603,17 @@ class Actor {
         }
         const { Dataset } = await import('@crawlee/core');
         const dataset = await Dataset.open();
+        const isDefault = dataset.id === this.config.get('defaultDatasetId');
+        // A priced default-dataset item event: rows are trimmed to the budget and
+        // counted like a charged push, so the spending limit is seen locally.
+        if (eventName === undefined
+            && !this.usesPushDataInterception(dataset)
+            && this.initialized
+            && isDefault
+            && this.chargingManager._isPayPerEvent()
+            && this.chargingManager._isPaidPerItem(undefined, true)) {
+            return this.pushDataWithExplicitCharging(dataset, item, undefined);
+        }
         if (eventName === undefined && !this.usesPushDataInterception(dataset)) {
             // Charge-limit safety net (also for local storage): this push is not
             // charged, so past the run's spending limit it is not stored.

package/dist/charging.d.ts

@@ -84,7 +84,24 @@ export interface ChargeResult {
     eventChargeLimitReached: boolean;
     chargedCount: number;
     chargeableWithinLimit: Record<string, number>;
+    /**
+     * `pushData(rows, { charges })` only: for each event whose charge came back
+     * short, how many events the limit (account balance, maxTotalChargeUsd,
+     * maxItems) covered. A row is saved only when all of its events are charged,
+     * so those events were given back, not charged. Empty when no charge came
+     * back short.
+     */
+    coveredCounts?: Record<string, number>;
+}
+/** @internal A charge of an atomic push that the limit accepted only in part (then given back). */
+export interface ShortCharge {
+    eventName: string;
+    requested: number;
+    covered: number;
+    limit: string;
 }
+/** @internal Log what an atomic push whose charge came back short really did. */
+export declare function logAtomicRefusal(shorts: ShortCharge[], rowsPushed: number, rowsSaved: number, givenBack: boolean): void;
 export interface ActorPricingInfo {
     pricingModel?: ActorRunPricingInfo['pricingModel'];
     maxTotalChargeUsd: number;
@@ -110,6 +127,12 @@ export declare class ChargingManager {
     private lastLimitMaxItems;
     /** Default-dataset rows refused after the limit (one warning, then a count). */
     private refusedDefaultDatasetItems;
+    /**
+     * @internal Set by pushDataWithCharges while it runs (it holds the operation
+     * lock): charges that come back short are collected here instead of being
+     * logged as "only N events were charged", because the atomic push gives them back.
+     */
+    _shortCharges: ShortCharge[] | undefined;
     /**
      * Prepaid-row balance: priced events charged through charge() minus rows
      * pushed to the default dataset without a per-row charge. An Actor that

package/dist/charging.js

@@ -10,6 +10,7 @@ exports.refuseDefaultDatasetPush = refuseDefaultDatasetPush;
 exports.isDefaultDatasetItemEvent = isDefaultDatasetItemEvent;
 exports.resolveDefaultDatasetItemEventName = resolveDefaultDatasetItemEventName;
 exports.hasDefaultDatasetItemEventInPricing = hasDefaultDatasetItemEventInPricing;
+exports.logAtomicRefusal = logAtomicRefusal;
 exports.mergeChargeResults = mergeChargeResults;
 exports.assertItemsSerializable = assertItemsSerializable;
 exports.pushDataAndCharge = pushDataAndCharge;
@@ -234,6 +235,32 @@ function hasDefaultDatasetItemEventInPricing(perEventPrices) {
 function isSyntheticEvent(eventName) {
     return isDefaultDatasetItemEvent(eventName);
 }
+function itemsPhrase(count) {
+    return count === 1
+        ? { what: 'this item', were: 'it was', them: 'it' }
+        : { what: `these ${count} items`, were: 'they were', them: 'them' };
+}
+/** @internal Log what an atomic push whose charge came back short really did. */
+function logAtomicRefusal(shorts, rowsPushed, rowsSaved, givenBack) {
+    const first = shorts[0];
+    if (!first)
+        return;
+    const limit = first.limit.replace(/^the end of /, '');
+    const { what, were, them } = itemsPhrase(rowsPushed);
+    const head = `[Scrapely] Saving ${what} needs ${first.requested} '${first.eventName}' event(s), but ${limit} covers only ${first.covered}`;
+    if (rowsSaved <= 0) {
+        console.log(givenBack
+            ? `${head}: ${were} not saved and nothing was charged for ${them}.`
+            : `${head}: ${were} not saved, but the ${first.covered} event(s) charged for ${them} could not be given back yet (see the warning about it).`);
+        return;
+    }
+    const left = rowsPushed - rowsSaved;
+    const rest = itemsPhrase(left);
+    const tail = givenBack
+        ? `nothing was charged for ${rest.them}`
+        : `what was charged for ${rest.them} could not be given back yet (see the warning about it)`;
+    console.log(`${head}: saved the first ${rowsSaved} item(s), charged in full; the other ${left} ${rest.were.split(' ').pop()} not saved and ${tail}.`);
+}
 function mergeChargeResults(a, b) {
     return {
         eventChargeLimitReached: a.eventChargeLimitReached || b.eventChargeLimitReached,
@@ -244,6 +271,17 @@ function mergeChargeResults(a, b) {
         ])),
     };
 }
+/**
+ * The run's spending limit (maxTotalChargeUsd), or Infinity when uncapped.
+ * 0 is a real limit (no fees), the same as on the platform; only a missing
+ * or non-numeric value means uncapped.
+ */
+function spendingLimitUsd(value) {
+    if (value == null || value === '')
+        return Infinity;
+    const usd = Number(value);
+    return Number.isFinite(usd) && usd >= 0 ? usd : Infinity;
+}
 /**
  * Pay-per-event charging for platform runs.
  * Reports charges via POST /v2/actor-runs/:runId/charge (Redis → worker billing).
@@ -262,6 +300,12 @@ class ChargingManager {
     lastLimitMaxItems;
     /** Default-dataset rows refused after the limit (one warning, then a count). */
     refusedDefaultDatasetItems = 0;
+    /**
+     * @internal Set by pushDataWithCharges while it runs (it holds the operation
+     * lock): charges that come back short are collected here instead of being
+     * logged as "only N events were charged", because the atomic push gives them back.
+     */
+    _shortCharges;
     /**
      * Prepaid-row balance: priced events charged through charge() minus rows
      * pushed to the default dataset without a per-row charge. An Actor that
@@ -577,7 +621,7 @@ class ChargingManager {
     constructor(configuration, apifyClient) {
         this.configuration = configuration;
         this.apifyClientSource = apifyClient;
-        this.maxTotalChargeUsd = configuration.get('maxTotalChargeUsd') || Infinity;
+        this.maxTotalChargeUsd = spendingLimitUsd(configuration.get('maxTotalChargeUsd'));
         this.isAtHome = !!configuration.get('isAtHome');
         this.actorRunId = configuration.get('actorRunId') || '';
     }
@@ -619,7 +663,7 @@ class ChargingManager {
             return {
                 pricingInfo: actorPricingInfo ? JSON.parse(actorPricingInfo) : undefined,
                 chargedEventCounts: JSON.parse(chargedEventCounts),
-                maxTotalChargeUsd: this.configuration.get('maxTotalChargeUsd') || Infinity,
+                maxTotalChargeUsd: spendingLimitUsd(this.configuration.get('maxTotalChargeUsd')),
             };
         }
         if (this.isAtHome) {
@@ -633,13 +677,13 @@ class ChargingManager {
             return {
                 pricingInfo: run.pricingInfo,
                 chargedEventCounts: run.chargedEventCounts,
-                maxTotalChargeUsd: run.options?.maxTotalChargeUsd || Infinity,
+                maxTotalChargeUsd: spendingLimitUsd(run.options?.maxTotalChargeUsd),
             };
         }
         return {
             pricingInfo: undefined,
             chargedEventCounts: {},
-            maxTotalChargeUsd: this.configuration.get('maxTotalChargeUsd') || Infinity,
+            maxTotalChargeUsd: spendingLimitUsd(this.configuration.get('maxTotalChargeUsd')),
         };
     }
     /**
@@ -1328,8 +1372,16 @@ class ChargingManager {
             });
         }
         if (chargedCount < count) {
-            const subject = count === 1 ? 'instance' : 'instances';
-            console.log(`[Scrapely] Charging ${count} ${subject} of '${normalizedEventName}' event would exceed ${this.limitName()} - only ${chargedCount} events were charged`);
+            if (this._shortCharges) {
+                // Inside an atomic push: this charge is given back unless the row is
+                // saved, so "only N events were charged" would not be true there.
+                // pushDataWithCharges logs what really happened instead.
+                this._shortCharges.push({ eventName: normalizedEventName, requested: count, covered: chargedCount, limit: this.limitName() });
+            }
+            else {
+                const subject = count === 1 ? 'instance' : 'instances';
+                console.log(`[Scrapely] Charging ${count} ${subject} of '${normalizedEventName}' event would exceed ${this.limitName()} - only ${chargedCount} events were charged`);
+            }
         }
         return {
             eventChargeLimitReached: this.calculateMaxEventChargeCountWithinLimit(normalizedEventName) <= 0,
@@ -1814,6 +1866,10 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
         // An item the storage would refuse fails here, before it is charged.
         assertItemsSerializable(rowsIn);
         validateItems?.(rowsIn);
+        // Charges of this push that came back short: given back, and logged once the
+        // outcome is known (logAtomicRefusal), not as "only N events were charged".
+        const shorts = [];
+        let givenBack = true;
         const undo = async (ops) => {
             for (const op of ops) {
                 if (op.charged <= 0)
@@ -1823,61 +1879,75 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
                 else if (!op.operationId)
                     chargingManager.applyAcceptedCount(op.name, op.charged, 0);
                 // Cancel / void; a refused void stays billed and becomes credit.
-                else
-                    await voidOrCredit(chargingManager, op.name, op.operationId, op.charged, op.uncertain);
+                else if ((await voidOrCredit(chargingManager, op.name, op.operationId, op.charged, op.uncertain)) !== 'voided')
+                    givenBack = false;
             }
         };
+        // Per event: how many events the limit covered in its last short answer.
+        const coveredCounts = () => Object.fromEntries(shorts.map((sc) => [sc.eventName, sc.covered]));
         const order = Object.entries(perRow).sort(([a], [b]) => Number(isSyntheticEvent(a)) - Number(isSyntheticEvent(b)) || a.localeCompare(b));
         let ops = [];
-        for (let attempt = 0; attempt < 6 && rows > 0; attempt++) {
-            ops = [];
-            let shortTo = null;
-            for (const [name, n] of order) {
-                const want = n * rows;
-                chargingManager._takeLastAcceptedOperationId?.();
-                let got;
-                if (takeVoidCredit(chargingManager, name, want) > 0) {
-                    got = want;
-                    ops.push({ name, charged: want, uncertain: false, credit: true });
-                }
-                else {
-                    try {
-                        const r = await chargingManager._chargeExclusive({ eventName: name, count: want });
-                        got = r.chargedCount;
-                        ops.push({ name, charged: got, operationId: chargingManager._takeLastAcceptedOperationId?.(), uncertain: false });
+        chargingManager._shortCharges = shorts;
+        try {
+            for (let attempt = 0; attempt < 6 && rows > 0; attempt++) {
+                ops = [];
+                let shortTo = null;
+                for (const [name, n] of order) {
+                    const want = n * rows;
+                    chargingManager._takeLastAcceptedOperationId?.();
+                    let got;
+                    if (takeVoidCredit(chargingManager, name, want) > 0) {
+                        got = want;
+                        ops.push({ name, charged: want, uncertain: false, credit: true });
                     }
-                    catch (err) {
-                        if (!(err instanceof ChargeUncertainError)) {
-                            await undo(ops);
-                            throw err;
+                    else {
+                        try {
+                            const r = await chargingManager._chargeExclusive({ eventName: name, count: want });
+                            got = r.chargedCount;
+                            ops.push({ name, charged: got, operationId: chargingManager._takeLastAcceptedOperationId?.(), uncertain: false });
+                        }
+                        catch (err) {
+                            if (!(err instanceof ChargeUncertainError)) {
+                                await undo(ops);
+                                throw err;
+                            }
+                            // Not confirmed: stays pending and is re-posted at exit, so it counts as accepted.
+                            got = err.count;
+                            ops.push({ name, charged: got, operationId: err.operationId, uncertain: true });
                         }
-                        // Not confirmed: stays pending and is re-posted at exit, so it counts as accepted.
-                        got = err.count;
-                        ops.push({ name, charged: got, operationId: err.operationId, uncertain: true });
+                    }
+                    if (got < want) {
+                        shortTo = Math.floor(got / n);
+                        break;
                     }
                 }
-                if (got < want) {
-                    shortTo = Math.floor(got / n);
+                if (shortTo === null)
                     break;
-                }
+                // An event came back short: void this attempt and retry with the whole
+                // rows the budget covers at the FULL per-row cost (sizing from the short
+                // event alone came back short again: appstore 64df0a3608d74af88, 0 saved).
+                // At least one single-row try: the last affordable row may still fit.
+                const attemptOps = ops.map((op) => ({ name: op.name, charged: op.charged }));
+                await undo(ops);
+                ops = [];
+                rows = Math.min(rows - 1, Math.max(chargingManager._rowsAfterShort(attemptOps, perRow), 1));
             }
-            if (shortTo === null)
-                break;
-            // An event came back short: void this attempt and retry with the whole
-            // rows the budget covers at the FULL per-row cost (sizing from the short
-            // event alone came back short again: appstore 64df0a3608d74af88, 0 saved).
-            // At least one single-row try: the last affordable row may still fit.
-            const attemptOps = ops.map((op) => ({ name: op.name, charged: op.charged }));
-            await undo(ops);
-            ops = [];
-            rows = Math.min(rows - 1, Math.max(chargingManager._rowsAfterShort(attemptOps, perRow), 1));
+        }
+        finally {
+            chargingManager._shortCharges = undefined;
         }
         if (rows <= 0 || ops.length === 0) {
             await undo(ops);
+            logAtomicRefusal(shorts, rowsIn.length, 0, givenBack);
             chargingManager._markChargeLimitHit();
             if (isDefaultDataset)
                 chargingManager._noteRefusedDefaultDatasetItems(rowsIn.length);
-            return { eventChargeLimitReached: true, chargedCount: 0, chargeableWithinLimit: chargingManager._chargeableWithinLimit() };
+            return {
+                eventChargeLimitReached: true,
+                chargedCount: 0,
+                chargeableWithinLimit: chargingManager._chargeableWithinLimit(),
+                coveredCounts: coveredCounts(),
+            };
         }
         const kept = batch ? rowsIn : rowsIn.slice(0, rows);
         if (kept.length < rowsIn.length) {
@@ -1894,10 +1964,12 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
                 await undo(ops);
             throw pushErr;
         }
+        logAtomicRefusal(shorts, rowsIn.length, kept.length, givenBack);
         return {
             eventChargeLimitReached: kept.length < rowsIn.length || chargingManager.isChargeLimitReached(),
             chargedCount: kept.length,
             chargeableWithinLimit: chargingManager._chargeableWithinLimit(),
+            coveredCounts: coveredCounts(),
         };
     });
 }
@@ -1919,6 +1991,9 @@ function pushCertainlyFailed(err) {
     for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) {
         if (e.outcomeUnknown === true)
             return false;
+        // The server could not tell whether the push was stored (a 503 with this type).
+        if (apiErrorTypeOf(e) === 'push-outcome-unknown')
+            return false;
     }
     if ((0, api_storage_client_js_1.isConnectFailure)(err))
         return true;

package/dist/storage/dataset-client.js

@@ -4,6 +4,7 @@ exports.DragonflyDatasetClient = void 0;
 const utils_js_1 = require("./utils.js");
 const billing_tracker_js_1 = require("./billing-tracker.js");
 const dataset_push_script_js_1 = require("./dataset-push-script.js");
+const ttl_manager_js_1 = require("./ttl-manager.js");
 function makeAsyncIterableResult(promiseFn) {
     const promise = promiseFn();
     const iterable = {
@@ -60,11 +61,13 @@ class DragonflyDatasetClient {
             const raw = await this.client.redis.hget(indexKey, `id:${this.id}`);
             if (raw) {
                 const data = JSON.parse(raw);
-                if (data.name)
-                    await this.client.redis.hdel(indexKey, `name:${data.name}`);
+                // Name pointers are scoped by owner (name:{userId}:{name}).
+                const ownerId = data.userId ?? this.client.billingContext?.userId;
+                await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, ownerId, data.name, this.id);
                 data.name = newFields.name;
                 await this.client.redis.hset(indexKey, `id:${this.id}`, JSON.stringify(data));
-                await this.client.redis.hset(indexKey, `name:${newFields.name}`, JSON.stringify(data));
+                if (ownerId && newFields.name)
+                    await this.client.redis.hset(indexKey, (0, utils_js_1.ownerNameField)(ownerId, newFields.name), JSON.stringify(data));
             }
         }
         await this.client.redis.hset(this.infoKey, 'modifiedAt', now.toISOString());
@@ -75,8 +78,7 @@ class DragonflyDatasetClient {
         await this.client.redis.del(this.infoKey, this.itemsKey);
         const indexKey = this.client.key('datasets:index');
         await this.client.redis.hdel(indexKey, `id:${this.id}`);
-        if (info?.name)
-            await this.client.redis.hdel(indexKey, `name:${info.name}`);
+        await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, info?.userId || this.client.billingContext?.userId, info?.name, this.id);
     }
     async downloadItems() {
         const result = await this._listItems();
@@ -216,6 +218,10 @@ class DragonflyDatasetClient {
         // separate round trips allowed concurrent pushes to allocate the same
         // slot indices, silently overwriting items.
         await (0, dataset_push_script_js_1.pushItemsAtomic)(this.client.redis, this.infoKey, this.itemsKey, serializedItems, totalBytes);
+        // The items hash is created by the first push: give it the dataset's expiry.
+        const expiry = this.client.redis.pipeline();
+        (0, ttl_manager_js_1.queueInheritExpiry)(expiry, this.infoKey, [this.itemsKey]);
+        await expiry.exec();
         // Track billing using client's billing context
         if (this.client.billingContext) {
             const billingTracker = new billing_tracker_js_1.BillingTracker(this.client);

package/dist/storage/dataset-collection-client.js

@@ -9,8 +9,10 @@ class DragonflyDatasetCollectionClient {
     }
     async getOrCreate(name) {
         const indexKey = this.client.key('datasets:index');
+        // Every storage belongs to the run's user; the name pointer is scoped by that user.
+        const userId = (0, utils_js_1.requireStorageOwner)(this.client.billingContext, 'dataset');
         if (name) {
-            const existing = await this.client.redis.hget(indexKey, `name:${name}`);
+            const existing = await this.client.redis.hget(indexKey, (0, utils_js_1.ownerNameField)(userId, name));
             if (existing)
                 return JSON.parse(existing);
         }
@@ -25,7 +27,7 @@ class DragonflyDatasetCollectionClient {
         };
         await this.client.redis.hset(indexKey, `id:${id}`, JSON.stringify(data));
         if (name)
-            await this.client.redis.hset(indexKey, `name:${name}`, JSON.stringify(data));
+            await this.client.redis.hset(indexKey, (0, utils_js_1.ownerNameField)(userId, name), JSON.stringify(data));
         const infoKey = this.client.key('dataset', id, 'info');
         const itemsKey = this.client.key('dataset', id, 'items');
         await this.client.redis.hset(infoKey, {
@@ -62,8 +64,7 @@ class DragonflyDatasetCollectionClient {
             const exists = await this.client.redis.exists(infoKey);
             if (!exists) {
                 await this.client.redis.hdel(indexKey, `id:${parsed.id}`);
-                if (parsed.name)
-                    await this.client.redis.hdel(indexKey, `name:${parsed.name}`);
+                await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, parsed.userId ?? this.client.billingContext?.userId, parsed.name, parsed.id);
                 continue;
             }
             const itemCount = parseInt(await this.client.redis.hget(infoKey, 'itemCount') || '0', 10);

package/dist/storage/key-value-store-client.js

@@ -4,6 +4,7 @@ exports.DragonflyKeyValueStoreClient = void 0;
 const node_stream_1 = require("node:stream");
 const utils_js_1 = require("./utils.js");
 const billing_tracker_js_1 = require("./billing-tracker.js");
+const ttl_manager_js_1 = require("./ttl-manager.js");
 const configuration_js_1 = require("../configuration.js");
 const kvs_iteration_js_1 = require("./kvs-iteration.js");
 function makeAsyncIterableListKeys(promiseFn) {
@@ -70,11 +71,13 @@ class DragonflyKeyValueStoreClient {
             const raw = await this.client.redis.hget(indexKey, `id:${this.id}`);
             if (raw) {
                 const data = JSON.parse(raw);
-                if (data.name)
-                    await this.client.redis.hdel(indexKey, `name:${data.name}`);
+                // Name pointers are scoped by owner (name:{userId}:{name}).
+                const ownerId = data.userId ?? this.client.billingContext?.userId;
+                await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, ownerId, data.name, this.id);
                 data.name = newFields.name;
                 await this.client.redis.hset(indexKey, `id:${this.id}`, JSON.stringify(data));
-                await this.client.redis.hset(indexKey, `name:${newFields.name}`, JSON.stringify(data));
+                if (ownerId && newFields.name)
+                    await this.client.redis.hset(indexKey, (0, utils_js_1.ownerNameField)(ownerId, newFields.name), JSON.stringify(data));
             }
         }
         await this.client.redis.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
@@ -85,8 +88,7 @@ class DragonflyKeyValueStoreClient {
         await this.client.redis.del(this.infoKey, this.recordsKey, this.metaKey);
         const indexKey = this.client.key('kvs:index');
         await this.client.redis.hdel(indexKey, `id:${this.id}`);
-        if (info?.name)
-            await this.client.redis.hdel(indexKey, `name:${info.name}`);
+        await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, info?.userId || this.client.billingContext?.userId, info?.name, this.id);
     }
     async recordExists(key) {
         return (await this.client.redis.hexists(this.recordsKey, key)) === 1;
@@ -168,6 +170,8 @@ class DragonflyKeyValueStoreClient {
             pipeline.hset(this.metaKey, key, JSON.stringify({ contentType }));
         }
         pipeline.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+        // Records/meta are created by the first write: give them the store's expiry.
+        (0, ttl_manager_js_1.queueInheritExpiry)(pipeline, this.infoKey, [this.recordsKey, this.metaKey]);
         await pipeline.exec();
         // Track billing using client's billing context
         if (this.client.billingContext) {

package/dist/storage/kv-store-collection-client.js

@@ -9,17 +9,15 @@ class DragonflyKeyValueStoreCollectionClient {
     }
     async getOrCreate(name) {
         const indexKey = this.client.key('kvs:index');
+        // Every storage belongs to the run's user; the name pointer is scoped by that user.
+        const userId = (0, utils_js_1.requireStorageOwner)(this.client.billingContext, 'key-value store');
         if (name) {
-            const existing = await this.client.redis.hget(indexKey, `name:${name}`);
+            const existing = await this.client.redis.hget(indexKey, (0, utils_js_1.ownerNameField)(userId, name));
             if (existing)
                 return JSON.parse(existing);
         }
         const id = (0, utils_js_1.generateId)();
         const now = (0, utils_js_1.nowISO)();
-        // Derive ownership from the active billing context (as the request-queue
-        // client does); the server authorizes against this persisted field.
-        // The legacy '1' placeholder applies only when no context is present.
-        const userId = this.client.billingContext?.userId ?? '1';
         const data = {
             id,
             name,
@@ -30,7 +28,7 @@ class DragonflyKeyValueStoreCollectionClient {
         };
         await this.client.redis.hset(indexKey, `id:${id}`, JSON.stringify(data));
         if (name)
-            await this.client.redis.hset(indexKey, `name:${name}`, JSON.stringify(data));
+            await this.client.redis.hset(indexKey, (0, utils_js_1.ownerNameField)(userId, name), JSON.stringify(data));
         const infoKey = this.client.key('kvs', id, 'info');
         const recordsKey = this.client.key('kvs', id, 'records');
         const metaKey = this.client.key('kvs', id, 'meta');
@@ -68,8 +66,7 @@ class DragonflyKeyValueStoreCollectionClient {
             const exists = await this.client.redis.exists(infoKey);
             if (!exists) {
                 await this.client.redis.hdel(indexKey, `id:${parsed.id}`);
-                if (parsed.name)
-                    await this.client.redis.hdel(indexKey, `name:${parsed.name}`);
+                await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, parsed.userId, parsed.name, parsed.id);
                 continue;
             }
             items.push(parsed);

package/dist/storage/log-manager.d.ts

@@ -1,5 +1,5 @@
 import type { DragonflyClient } from './dragonfly-client.js';
-import type { UserPlan } from './ttl-manager.js';
+import { type UserPlan } from './ttl-manager.js';
 /**
  * Log configuration
  */

package/dist/storage/log-manager.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LogManager = void 0;
+const ttl_manager_js_1 = require("./ttl-manager.js");
 const run_metadata_js_1 = require("./run-metadata.js");
 const DEFAULT_LOG_CONFIG = {
     maxLogSize: 5 * 1024 * 1024, // 5MB
@@ -89,10 +90,14 @@ class LogManager {
             // a valid code-point boundary and never store a broken sequence.
             const alignedOffset = LogManager.alignUtf8Boundary(trailingBuf, 0);
             const safeTrailing = trailingBuf.subarray(alignedOffset);
-            // Overwrite the key with only the safe trailing portion
-            await this.client.redis.set(key, safeTrailing);
-            // Mark log as truncated (NX — only set once; ignore if already set)
+            // Overwrite the key with only the safe trailing portion (keeping the log's expiry)
+            await this.client.redis.set(key, safeTrailing, 'KEEPTTL');
+            // Mark log as truncated (NX — only set once; ignore if already set). The marker
+            // is created here, after setLogTtl ran: it takes the log's expiry.
             await this.client.redis.set(this.getTruncatedKey(runId), '1', 'NX');
+            const expiry = this.client.redis.pipeline();
+            (0, ttl_manager_js_1.queueInheritExpiry)(expiry, key, [this.getTruncatedKey(runId)]);
+            await expiry.exec();
         }
     }
     /**
@@ -118,7 +123,7 @@ class LogManager {
         const key = this.getLogKey(runId);
         const ttl = this.client.ttlManager.getTtl('logs', plan);
         if (ttl !== null && ttl > 0) {
-            await this.client.ttlManager.setExpiry(this.client.redis, [key], ttl);
+            await this.client.ttlManager.setExpiry(this.client.redis, [key, this.getTruncatedKey(runId)], ttl);
         }
     }
     /**

package/dist/storage/request-queue-client.d.ts

@@ -20,6 +20,15 @@ export interface ListRequestsResult {
 import type { DragonflyClient } from './dragonfly-client.js';
 import type { RequestQueueOptions } from '@crawlee/types';
 import { type RequestQueueRequest } from './rq-list-requests.js';
+/** Longest lock, in seconds (same as the server's MAX_LOCK_SECS). */
+export declare const MAX_LOCK_SECS = 86400;
+/** lockSecs as a whole number of seconds from 1 to MAX_LOCK_SECS. */
+export declare function clampLockSecs(lockSecs: number): number;
+/**
+ * Largest stored request a head read loads (the server's MAX_READABLE_STORED_REQUEST_BYTES):
+ * a larger one is left out of the head instead of being read into memory.
+ */
+export declare const MAX_HEAD_REQUEST_BYTES: number;
 export declare class DragonflyRequestQueueClient implements RequestQueueClient {
     private client;
     private id;
@@ -71,6 +80,8 @@ export declare class DragonflyRequestQueueClient implements RequestQueueClient {
      * @param options.clientKey - Client identifier for hadMultipleClients tracking
      */
     listRequests(options?: ListRequestsOptions): Promise<ListRequestsResult>;
+    /** The ids whose stored request is at most MAX_HEAD_REQUEST_BYTES (HSTRLEN, nothing loaded). */
+    private withinHeadSize;
     listAndLockHead(options: ListAndLockOptions & {
         clientKey?: string;
     }): Promise<ListAndLockHeadResult>;

package/dist/storage/request-queue-client.js

@@ -1,11 +1,34 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DragonflyRequestQueueClient = void 0;
+exports.DragonflyRequestQueueClient = exports.MAX_HEAD_REQUEST_BYTES = exports.MAX_LOCK_SECS = void 0;
+exports.clampLockSecs = clampLockSecs;
 const utils_js_1 = require("./utils.js");
 const billing_tracker_js_1 = require("./billing-tracker.js");
 const rq_lock_script_js_1 = require("./rq-lock-script.js");
 const rq_list_requests_js_1 = require("./rq-list-requests.js");
+const ttl_manager_js_1 = require("./ttl-manager.js");
 const node_crypto_1 = require("node:crypto");
+/**
+ * Lock owners are stored as "clientKey:runId", so a clientKey holding ':'
+ * could present another run's owner value. Same rule as the server.
+ */
+function assertClientKey(clientKey) {
+    if (clientKey.includes(':'))
+        throw new Error("clientKey must not contain ':'");
+    return clientKey;
+}
+/** Longest lock, in seconds (same as the server's MAX_LOCK_SECS). */
+exports.MAX_LOCK_SECS = 86_400;
+/** lockSecs as a whole number of seconds from 1 to MAX_LOCK_SECS. */
+function clampLockSecs(lockSecs) {
+    const n = Math.floor(Number(lockSecs));
+    return Number.isFinite(n) ? Math.min(Math.max(n, 1), exports.MAX_LOCK_SECS) : 1;
+}
+/**
+ * Largest stored request a head read loads (the server's MAX_READABLE_STORED_REQUEST_BYTES):
+ * a larger one is left out of the head instead of being read into memory.
+ */
+exports.MAX_HEAD_REQUEST_BYTES = 10 * 1024 * 1024 + 64 * 1024;
 class DragonflyRequestQueueClient {
     client;
     id;
@@ -18,7 +41,7 @@ class DragonflyRequestQueueClient {
     constructor(client, id, options) {
         this.client = client;
         this.id = id;
-        this.clientKey = options?.clientKey || (0, node_crypto_1.randomUUID)().replace(/-/g, '').slice(0, 16);
+        this.clientKey = assertClientKey(options?.clientKey || (0, node_crypto_1.randomUUID)().replace(/-/g, '').slice(0, 16));
         this.infoKey = client.key('rq', id, 'info');
         this.requestsKey = client.key('rq', id, 'requests');
         this.pendingKey = client.key('rq', id, 'pending');
@@ -119,11 +142,13 @@ class DragonflyRequestQueueClient {
             const raw = await this.client.redis.hget(indexKey, `id:${this.id}`);
             if (raw) {
                 const data = JSON.parse(raw);
-                if (data.name)
-                    await this.client.redis.hdel(indexKey, `name:${data.name}`);
+                // Name pointers are scoped by owner (name:{userId}:{name}).
+                const ownerId = data.userId ?? this.client.billingContext?.userId;
+                await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, ownerId, data.name, this.id);
                 data.name = newFields.name;
                 await this.client.redis.hset(indexKey, `id:${this.id}`, JSON.stringify(data));
-                await this.client.redis.hset(indexKey, `name:${newFields.name}`, JSON.stringify(data));
+                if (ownerId && newFields.name)
+                    await this.client.redis.hset(indexKey, (0, utils_js_1.ownerNameField)(ownerId, newFields.name), JSON.stringify(data));
             }
         }
         await this.client.redis.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
@@ -134,8 +159,7 @@ class DragonflyRequestQueueClient {
         await this.client.redis.del(this.infoKey, this.requestsKey, this.pendingKey, this.lockedKey, this.lockOwnersKey);
         const indexKey = this.client.key('rq:index');
         await this.client.redis.hdel(indexKey, `id:${this.id}`);
-        if (info?.name)
-            await this.client.redis.hdel(indexKey, `name:${info.name}`);
+        await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, info?.userId || this.client.billingContext?.userId, info?.name, this.id);
     }
     async addRequest(request, options) {
         const requestId = request.id || (0, utils_js_1.uniqueKeyToRequestId)(request.uniqueKey);
@@ -156,6 +180,8 @@ class DragonflyRequestQueueClient {
         pipeline.hincrby(this.infoKey, 'totalRequestCount', 1);
         pipeline.hincrby(this.infoKey, 'pendingRequestCount', 1);
         pipeline.hset(this.infoKey, 'modifiedAt', (0, utils_js_1.nowISO)().toISOString());
+        // Requests/pending are created by the first add: give them the queue's expiry.
+        (0, ttl_manager_js_1.queueInheritExpiry)(pipeline, this.infoKey, [this.requestsKey, this.pendingKey]);
         await pipeline.exec();
         // Track write and storage bytes for billing
         const bytes = billing_tracker_js_1.BillingTracker.estimateRequestBytes(stored);
@@ -289,7 +315,7 @@ class DragonflyRequestQueueClient {
         const activeLocks = new Set(at(1));
         // Filter out locked items
         const items = [];
-        const candidates = ids.filter((id) => !activeLocks.has(id)).slice(0, limit);
+        const candidates = await this.withinHeadSize(ids.filter((id) => !activeLocks.has(id)).slice(0, limit));
         if (candidates.length > 0) {
             const raws = await this.client.redis.hmget(this.requestsKey, ...candidates);
             for (const raw of raws) {
@@ -408,11 +434,22 @@ class DragonflyRequestQueueClient {
         }
         return result;
     }
+    /** The ids whose stored request is at most MAX_HEAD_REQUEST_BYTES (HSTRLEN, nothing loaded). */
+    async withinHeadSize(ids) {
+        if (ids.length === 0)
+            return ids;
+        const pipeline = this.client.redis.pipeline();
+        for (const id of ids)
+            pipeline.hstrlen(this.requestsKey, id);
+        const sizes = (await pipeline.exec()) ?? [];
+        return ids.filter((_, i) => Number(sizes[i]?.[1] ?? 0) <= exports.MAX_HEAD_REQUEST_BYTES);
+    }
     async listAndLockHead(options) {
         const limit = Math.min(options.limit ?? 25, 25);
         const now = Date.now();
-        const lockUntil = now + options.lockSecs * 1000;
-        const clientKey = options.clientKey || this.clientKey;
+        const lockSecs = clampLockSecs(options.lockSecs);
+        const lockUntil = now + lockSecs * 1000;
+        const clientKey = assertClientKey(options.clientKey || this.clientKey);
         const runId = this.client.billingContext?.runId || null;
         const ownerValue = runId ? `${clientKey}:${runId}` : clientKey;
         const { lockedIds, activeLockCount } = await (0, rq_lock_script_js_1.listAndLockHeadAtomic)(this.client.redis, {
@@ -425,11 +462,19 @@ class DragonflyRequestQueueClient {
             lockUntil,
             limit,
             ownerValue,
+            maxRequestBytes: exports.MAX_HEAD_REQUEST_BYTES,
         });
         const items = [];
         if (lockedIds.length > 0) {
+            // The locked set and owners hash are created by the first lock: give them the queue's expiry.
+            const expiry = this.client.redis.pipeline();
+            (0, ttl_manager_js_1.queueInheritExpiry)(expiry, this.infoKey, [this.lockedKey, this.lockOwnersKey]);
+            await expiry.exec();
+        }
+        const readable = await this.withinHeadSize(lockedIds);
+        if (readable.length > 0) {
             // Single HMGET instead of one HGET round-trip per locked ID
-            const raws = await this.client.redis.hmget(this.requestsKey, ...lockedIds);
+            const raws = await this.client.redis.hmget(this.requestsKey, ...readable);
             for (const raw of raws) {
                 if (!raw)
                     continue;
@@ -453,7 +498,7 @@ class DragonflyRequestQueueClient {
             limit,
             queueModifiedAt: modifiedAt ? new Date(modifiedAt) : (0, utils_js_1.nowISO)(),
             hadMultipleClients,
-            lockSecs: options.lockSecs,
+            lockSecs,
             items,
             // From the atomic lock script: active locks held by ANY client
             // (including this call's own).
@@ -485,7 +530,7 @@ class DragonflyRequestQueueClient {
         if (owner && !(0, rq_lock_script_js_1.matchesLockOwner)(owner, this.clientKey, runId)) {
             throw new Error(`Request with ID ${id} is not locked by this client in queue ${this.id}`);
         }
-        const lockUntil = now + options.lockSecs * 1000;
+        const lockUntil = now + clampLockSecs(options.lockSecs) * 1000;
         await this.client.redis.zadd(this.lockedKey, String(lockUntil), id);
         return { lockExpiresAt: new Date(lockUntil) };
     }
@@ -512,7 +557,7 @@ class DragonflyRequestQueueClient {
             lockOwnersKey: this.lockOwnersKey,
             pendingKey: this.pendingKey,
         }, {
-            clientKey: options.clientKey,
+            clientKey: assertClientKey(options.clientKey),
             forefront: options.forefront,
             runId: options.runId,
         });

package/dist/storage/rq-collection-client.js

@@ -9,18 +9,15 @@ class DragonflyRequestQueueCollectionClient {
     }
     async getOrCreate(name) {
         const indexKey = this.client.key('rq:index');
+        // Every storage belongs to the run's user; the name pointer is scoped by that user.
+        const userId = (0, utils_js_1.requireStorageOwner)(this.client.billingContext, 'request queue');
         if (name) {
-            const existing = await this.client.redis.hget(indexKey, `name:${name}`);
+            const existing = await this.client.redis.hget(indexKey, (0, utils_js_1.ownerNameField)(userId, name));
             if (existing)
                 return JSON.parse(existing);
         }
         const id = (0, utils_js_1.generateId)();
         const now = (0, utils_js_1.nowISO)();
-        // Real userId when billing context is active; legacy placeholder otherwise.
-        // Matches the KVS collection client convention — the server's storage usage
-        // and TTL jobs match on this field to attribute storages to users, so an
-        // empty value would orphan the queue from per-user accounting.
-        const userId = this.client.billingContext?.userId ?? '1';
         const data = {
             id,
             name,
@@ -36,7 +33,7 @@ class DragonflyRequestQueueCollectionClient {
         };
         await this.client.redis.hset(indexKey, `id:${id}`, JSON.stringify(data));
         if (name)
-            await this.client.redis.hset(indexKey, `name:${name}`, JSON.stringify(data));
+            await this.client.redis.hset(indexKey, (0, utils_js_1.ownerNameField)(userId, name), JSON.stringify(data));
         const infoKey = this.client.key('rq', id, 'info');
         const requestsKey = this.client.key('rq', id, 'requests');
         const pendingKey = this.client.key('rq', id, 'pending');
@@ -77,8 +74,7 @@ class DragonflyRequestQueueCollectionClient {
             const exists = await this.client.redis.exists(infoKey);
             if (!exists) {
                 await this.client.redis.hdel(indexKey, `id:${parsed.id}`);
-                if (parsed.name)
-                    await this.client.redis.hdel(indexKey, `name:${parsed.name}`);
+                await (0, utils_js_1.releaseOwnerName)(this.client.redis, indexKey, parsed.userId, parsed.name, parsed.id);
                 continue;
             }
             items.push(parsed);

package/dist/storage/rq-lock-script.d.ts

@@ -9,7 +9,7 @@ import type Redis from 'ioredis';
  * the one calling the endpoint or a different one"). Callers get the flag from
  * this same atomic reply instead of re-reading the whole locked + pending sets.
  */
-export declare const LIST_AND_LOCK_HEAD_LUA = "\nlocal pendingKey = KEYS[1]\nlocal lockedKey = KEYS[2]\nlocal lockOwnersKey = KEYS[3]\nlocal requestsKey = KEYS[4]\nlocal now = tonumber(ARGV[1])\nlocal lockUntil = tonumber(ARGV[2])\nlocal limit = tonumber(ARGV[3])\nlocal ownerValue = ARGV[4]\nlocal scanLimit = tonumber(ARGV[5])\n\nredis.call('ZREMRANGEBYSCORE', lockedKey, '-inf', now)\n\nlocal pendingIds = redis.call('ZRANGE', pendingKey, 0, scanLimit - 1)\nlocal lockedIds = {}\n\nfor _, id in ipairs(pendingIds) do\n  if #lockedIds >= limit then break end\n  local lockScore = redis.call('ZSCORE', lockedKey, id)\n  if not lockScore or tonumber(lockScore) <= now then\n    if redis.call('HEXISTS', requestsKey, id) == 1 then\n      redis.call('ZADD', lockedKey, lockUntil, id)\n      redis.call('HSET', lockOwnersKey, id, ownerValue)\n      table.insert(lockedIds, id)\n    end\n  end\nend\n\n-- Everything left in lockedKey is an active lock (score > now): the purge\n-- above removed the expired ones and we just added ours. An O(1) ZCARD here\n-- replaces the callers' old tail of ZRANGEBYSCORE(locked) + full\n-- ZRANGE(pending, 0, -1) scans on every head fetch.\nreturn { lockedIds, redis.call('ZCARD', lockedKey) }\n";
+export declare const LIST_AND_LOCK_HEAD_LUA = "\nlocal pendingKey = KEYS[1]\nlocal lockedKey = KEYS[2]\nlocal lockOwnersKey = KEYS[3]\nlocal requestsKey = KEYS[4]\nlocal now = tonumber(ARGV[1])\nlocal lockUntil = tonumber(ARGV[2])\nlocal limit = tonumber(ARGV[3])\nlocal ownerValue = ARGV[4]\nlocal scanLimit = tonumber(ARGV[5])\n-- Requests larger than this are never returned by the head, so they are not locked either.\nlocal maxRequestBytes = tonumber(ARGV[6])\n\n-- Expired locks leave lockedKey here, and their owner entries go with them:\n-- otherwise lockOwners keeps one entry per request ever locked.\nlocal expiredIds = redis.call('ZRANGEBYSCORE', lockedKey, '-inf', now)\nfor i = 1, #expiredIds, 1000 do\n  redis.call('HDEL', lockOwnersKey, unpack(expiredIds, i, math.min(i + 999, #expiredIds)))\nend\nredis.call('ZREMRANGEBYSCORE', lockedKey, '-inf', now)\n\nlocal pendingIds = redis.call('ZRANGE', pendingKey, 0, scanLimit - 1)\nlocal lockedIds = {}\n\nfor _, id in ipairs(pendingIds) do\n  if #lockedIds >= limit then break end\n  local lockScore = redis.call('ZSCORE', lockedKey, id)\n  if not lockScore or tonumber(lockScore) <= now then\n    if redis.call('HEXISTS', requestsKey, id) == 1\n      and (not maxRequestBytes or redis.call('HSTRLEN', requestsKey, id) <= maxRequestBytes) then\n      redis.call('ZADD', lockedKey, lockUntil, id)\n      redis.call('HSET', lockOwnersKey, id, ownerValue)\n      table.insert(lockedIds, id)\n    end\n  end\nend\n\n-- Everything left in lockedKey is an active lock (score > now): the purge\n-- above removed the expired ones and we just added ours. An O(1) ZCARD here\n-- replaces the callers' old tail of ZRANGEBYSCORE(locked) + full\n-- ZRANGE(pending, 0, -1) scans on every head fetch.\nreturn { lockedIds, redis.call('ZCARD', lockedKey) }\n";
 export interface ListAndLockHeadKeys {
     pendingKey: string;
     lockedKey: string;
@@ -23,6 +23,8 @@ export interface ListAndLockHeadParams {
     ownerValue: string;
     /** How many pending IDs to scan (should be >= limit). */
     scanLimit?: number;
+    /** Skip (don't lock) requests stored larger than this many bytes. */
+    maxRequestBytes?: number;
 }
 export interface ListAndLockHeadAtomicResult {
     /** IDs locked by this call, in queue-head order. */
@@ -36,14 +38,18 @@ export interface ListAndLockHeadAtomicResult {
 export declare function listAndLockHeadAtomic(redis: Redis, keys: ListAndLockHeadKeys, params: ListAndLockHeadParams): Promise<ListAndLockHeadAtomicResult>;
 /** Returns true if lockOwners entry matches the caller's clientKey (and optional runId). */
 export declare function matchesLockOwner(owner: string, clientKey: string, runId?: string | null): boolean;
+/** Locked requests UNLOCK_REQUESTS_LUA reads per page. */
+export declare const UNLOCK_SCAN_PAGE = 1000;
 /**
- * Atomic unlock-by-owner: scans lockOwners, unlocks every request whose owner
- * matches the clientKey (and optional runId) using the same rules as the old
- * per-request loop, optionally re-queueing unlocked requests at the forefront.
- * Returns the number of unlocked requests. One round-trip, no race window
- * between ZREM and HDEL, and the whole owners hash never leaves the server.
+ * Atomic unlock-by-owner: walks the locked set a page at a time by rank,
+ * unlocks every request whose owner matches the clientKey (and optional
+ * runId) using the same rules as the old per-request loop, optionally
+ * re-queueing unlocked requests at the forefront. Returns the number of
+ * unlocked requests. One round-trip, no race window between ZREM and HDEL,
+ * and neither the owners hash nor the locked set is read whole (pages of
+ * UNLOCK_SCAN_PAGE, ARGV[4]).
  */
-export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n  expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n  local requestId = flat[i]\n  local owner = flat[i + 1]\n  -- In an Actor run, unlock both (1) requests locked by this\n  -- specific run (clientKey:runId) and (2) requests locked by the same\n  -- clientKey without run context (clientKey), but NOT requests locked by the\n  -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n  if owner == expectedOwner or owner == clientKey\n      or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then\n    redis.call('ZREM', lockedKey, requestId)\n    redis.call('HDEL', lockOwnersKey, requestId)\n    if forefront == '1' then\n      redis.call('ZADD', pendingKey, 0, requestId)\n    end\n    unlockedCount = unlockedCount + 1\n  end\nend\n\nreturn unlockedCount\n";
+export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\nlocal pageSize = tonumber(ARGV[4])\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n  expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal unlockedCount = 0\nlocal start = 0\n\nwhile true do\n  local ids = redis.call('ZRANGE', lockedKey, start, start + pageSize - 1)\n  if #ids == 0 then break end\n  local owners = redis.call('HMGET', lockOwnersKey, unpack(ids))\n  local removed = 0\n  for i, requestId in ipairs(ids) do\n    local owner = owners[i]\n    -- In an Actor run, unlock both (1) requests locked by this\n    -- specific run (clientKey:runId) and (2) requests locked by the same\n    -- clientKey without run context (clientKey), but NOT requests locked by the\n    -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n    -- Exact clientKey, or this run's clientKey:runId. An empty runId must not\n    -- unlock every owner that merely starts with \"clientKey:\".\n    if owner and (owner == expectedOwner or owner == clientKey) then\n      redis.call('ZREM', lockedKey, requestId)\n      redis.call('HDEL', lockOwnersKey, requestId)\n      if forefront == '1' then\n        redis.call('ZADD', pendingKey, 0, requestId)\n      end\n      unlockedCount = unlockedCount + 1\n      removed = removed + 1\n    end\n  end\n  if #ids < pageSize then break end\n  -- Removed locks were in this page, so the next unread lock moved up by that many.\n  start = start + #ids - removed\nend\n\nreturn unlockedCount\n";
 export interface UnlockRequestsKeys {
     lockedKey: string;
     lockOwnersKey: string;

package/dist/storage/rq-lock-script.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UNLOCK_REQUESTS_LUA = exports.LIST_AND_LOCK_HEAD_LUA = void 0;
+exports.UNLOCK_REQUESTS_LUA = exports.UNLOCK_SCAN_PAGE = exports.LIST_AND_LOCK_HEAD_LUA = void 0;
 exports.listAndLockHeadAtomic = listAndLockHeadAtomic;
 exports.matchesLockOwner = matchesLockOwner;
 exports.unlockRequestsAtomic = unlockRequestsAtomic;
@@ -24,7 +24,15 @@ local lockUntil = tonumber(ARGV[2])
 local limit = tonumber(ARGV[3])
 local ownerValue = ARGV[4]
 local scanLimit = tonumber(ARGV[5])
+-- Requests larger than this are never returned by the head, so they are not locked either.
+local maxRequestBytes = tonumber(ARGV[6])
 
+-- Expired locks leave lockedKey here, and their owner entries go with them:
+-- otherwise lockOwners keeps one entry per request ever locked.
+local expiredIds = redis.call('ZRANGEBYSCORE', lockedKey, '-inf', now)
+for i = 1, #expiredIds, 1000 do
+  redis.call('HDEL', lockOwnersKey, unpack(expiredIds, i, math.min(i + 999, #expiredIds)))
+end
 redis.call('ZREMRANGEBYSCORE', lockedKey, '-inf', now)
 
 local pendingIds = redis.call('ZRANGE', pendingKey, 0, scanLimit - 1)
@@ -34,7 +42,8 @@ for _, id in ipairs(pendingIds) do
   if #lockedIds >= limit then break end
   local lockScore = redis.call('ZSCORE', lockedKey, id)
   if not lockScore or tonumber(lockScore) <= now then
-    if redis.call('HEXISTS', requestsKey, id) == 1 then
+    if redis.call('HEXISTS', requestsKey, id) == 1
+      and (not maxRequestBytes or redis.call('HSTRLEN', requestsKey, id) <= maxRequestBytes) then
       redis.call('ZADD', lockedKey, lockUntil, id)
       redis.call('HSET', lockOwnersKey, id, ownerValue)
       table.insert(lockedIds, id)
@@ -50,7 +59,7 @@ return { lockedIds, redis.call('ZCARD', lockedKey) }
 `;
 async function listAndLockHeadAtomic(redis, keys, params) {
     const scanLimit = params.scanLimit ?? Math.max(params.limit * 4, 100);
-    const result = await redis.eval(exports.LIST_AND_LOCK_HEAD_LUA, 4, keys.pendingKey, keys.lockedKey, keys.lockOwnersKey, keys.requestsKey, String(params.now), String(params.lockUntil), String(params.limit), params.ownerValue, String(scanLimit));
+    const result = await redis.eval(exports.LIST_AND_LOCK_HEAD_LUA, 4, keys.pendingKey, keys.lockedKey, keys.lockOwnersKey, keys.requestsKey, String(params.now), String(params.lockUntil), String(params.limit), params.ownerValue, String(scanLimit), ...(params.maxRequestBytes !== undefined ? [String(params.maxRequestBytes)] : []));
     // Expected reply shape: [string[], number]. Tolerate degraded replies the
     // same way the old string[] version did (null / JSON-stringified table).
     if (!Array.isArray(result) || result.length < 2) {
@@ -76,12 +85,16 @@ function matchesLockOwner(owner, clientKey, runId) {
     const expectedOwner = runId ? `${clientKey}:${runId}` : clientKey;
     return owner === expectedOwner || owner === clientKey;
 }
+/** Locked requests UNLOCK_REQUESTS_LUA reads per page. */
+exports.UNLOCK_SCAN_PAGE = 1000;
 /**
- * Atomic unlock-by-owner: scans lockOwners, unlocks every request whose owner
- * matches the clientKey (and optional runId) using the same rules as the old
- * per-request loop, optionally re-queueing unlocked requests at the forefront.
- * Returns the number of unlocked requests. One round-trip, no race window
- * between ZREM and HDEL, and the whole owners hash never leaves the server.
+ * Atomic unlock-by-owner: walks the locked set a page at a time by rank,
+ * unlocks every request whose owner matches the clientKey (and optional
+ * runId) using the same rules as the old per-request loop, optionally
+ * re-queueing unlocked requests at the forefront. Returns the number of
+ * unlocked requests. One round-trip, no race window between ZREM and HDEL,
+ * and neither the owners hash nor the locked set is read whole (pages of
+ * UNLOCK_SCAN_PAGE, ARGV[4]).
  */
 exports.UNLOCK_REQUESTS_LUA = `
 local lockedKey = KEYS[1]
@@ -90,38 +103,47 @@ local pendingKey = KEYS[3]
 local clientKey = ARGV[1]
 local runId = ARGV[2]
 local forefront = ARGV[3]
+local pageSize = tonumber(ARGV[4])
 
 local expectedOwner = clientKey
 if runId ~= '' then
   expectedOwner = clientKey .. ':' .. runId
 end
 
-local clientKeyPrefix = clientKey .. ':'
-
-local flat = redis.call('HGETALL', lockOwnersKey)
 local unlockedCount = 0
+local start = 0
 
-for i = 1, #flat, 2 do
-  local requestId = flat[i]
-  local owner = flat[i + 1]
-  -- In an Actor run, unlock both (1) requests locked by this
-  -- specific run (clientKey:runId) and (2) requests locked by the same
-  -- clientKey without run context (clientKey), but NOT requests locked by the
-  -- same clientKey in DIFFERENT runs (clientKey:otherRunId).
-  if owner == expectedOwner or owner == clientKey
-      or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then
-    redis.call('ZREM', lockedKey, requestId)
-    redis.call('HDEL', lockOwnersKey, requestId)
-    if forefront == '1' then
-      redis.call('ZADD', pendingKey, 0, requestId)
+while true do
+  local ids = redis.call('ZRANGE', lockedKey, start, start + pageSize - 1)
+  if #ids == 0 then break end
+  local owners = redis.call('HMGET', lockOwnersKey, unpack(ids))
+  local removed = 0
+  for i, requestId in ipairs(ids) do
+    local owner = owners[i]
+    -- In an Actor run, unlock both (1) requests locked by this
+    -- specific run (clientKey:runId) and (2) requests locked by the same
+    -- clientKey without run context (clientKey), but NOT requests locked by the
+    -- same clientKey in DIFFERENT runs (clientKey:otherRunId).
+    -- Exact clientKey, or this run's clientKey:runId. An empty runId must not
+    -- unlock every owner that merely starts with "clientKey:".
+    if owner and (owner == expectedOwner or owner == clientKey) then
+      redis.call('ZREM', lockedKey, requestId)
+      redis.call('HDEL', lockOwnersKey, requestId)
+      if forefront == '1' then
+        redis.call('ZADD', pendingKey, 0, requestId)
+      end
+      unlockedCount = unlockedCount + 1
+      removed = removed + 1
     end
-    unlockedCount = unlockedCount + 1
   end
+  if #ids < pageSize then break end
+  -- Removed locks were in this page, so the next unread lock moved up by that many.
+  start = start + #ids - removed
 end
 
 return unlockedCount
 `;
 async function unlockRequestsAtomic(redis, keys, params) {
-    const result = await redis.eval(exports.UNLOCK_REQUESTS_LUA, 3, keys.lockedKey, keys.lockOwnersKey, keys.pendingKey, params.clientKey, params.runId ?? '', params.forefront ? '1' : '0');
+    const result = await redis.eval(exports.UNLOCK_REQUESTS_LUA, 3, keys.lockedKey, keys.lockOwnersKey, keys.pendingKey, params.clientKey, params.runId ?? '', params.forefront ? '1' : '0', String(exports.UNLOCK_SCAN_PAGE));
     return typeof result === 'number' ? result : parseInt(String(result), 10) || 0;
 }

package/dist/storage/run-metadata.js

@@ -208,6 +208,12 @@ class RunMetadataManager {
             ttlPipeline.ttl(this.getRunKey(runId));
         }
         const ttlResults = await ttlPipeline.exec();
+        // Runs whose hash is gone (TTL = -2: expired or deleted outside delete()) leave the
+        // sorted set here, so it doesn't keep one entry per run ever created.
+        const staleIds = olderIds.filter((_, i) => ttlResults?.[i]?.[1] === -2);
+        if (staleIds.length > 0) {
+            await this.client.redis.zrem(userRunsKey, ...staleIds);
+        }
         // Collect only the runs that were demoted (exist but have no expiry, TTL = -1)
         const demotedIds = olderIds.filter((_, i) => ttlResults?.[i]?.[1] === -1);
         if (demotedIds.length === 0) {

package/dist/storage/ttl-manager.d.ts

@@ -1,4 +1,7 @@
 import type Redis from 'ioredis';
+import type { ChainableCommander } from 'ioredis';
+/** Queue INHERIT_EXPIRY_LUA: the lazily created `keys` expire with `infoKey` (nothing when it has no expiry). */
+export declare function queueInheritExpiry(pipeline: ChainableCommander, infoKey: string, keys: string[]): void;
 /**
  * TTL configuration in seconds
  */

package/dist/storage/ttl-manager.js

@@ -1,6 +1,26 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TtlManager = exports.DEFAULT_TTL_CONFIG = void 0;
+exports.queueInheritExpiry = queueInheritExpiry;
+/**
+ * KEYS[1]: a storage's info key; KEYS[2..]: its data keys. Gives each data key the info key's
+ * remaining expiry (NX: one that has an expiry keeps it). An expiry set at creation is a no-op
+ * on data keys that don't exist yet, so a key created lazily on the first write never expired.
+ */
+const INHERIT_EXPIRY_LUA = `
+local ttl = redis.call('PTTL', KEYS[1])
+if ttl <= 0 then return 0 end
+for i = 2, #KEYS do
+  redis.call('PEXPIRE', KEYS[i], ttl, 'NX')
+end
+return 1
+`;
+/** Queue INHERIT_EXPIRY_LUA: the lazily created `keys` expire with `infoKey` (nothing when it has no expiry). */
+function queueInheritExpiry(pipeline, infoKey, keys) {
+    if (keys.length === 0)
+        return;
+    pipeline.eval(INHERIT_EXPIRY_LUA, keys.length + 1, infoKey, ...keys);
+}
 /**
  * Default TTL configuration
  */

package/dist/storage/utils.d.ts

@@ -9,3 +9,22 @@ export declare function nowISO(): Date;
  * segments even when they are percent-encoded.
  */
 export declare function pathSegment(value: string): string;
+/**
+ * Field of a named storage in the global `{prefix}datasets:index` /
+ * `kvs:index` / `rq:index` hash: scoped by owner (`name:{userId}:{name}`, the
+ * same field the server writes), so two accounts using one name never share
+ * or overwrite each other's pointer.
+ */
+export declare function ownerNameField(userId: string, name: string): string;
+/** The owner a new storage is created for: the active billing context's user. */
+export declare function requireStorageOwner(billingContext: {
+    userId?: string;
+} | undefined, kind: string): string;
+/**
+ * Remove an owner's name pointer, but only while it still points at `storageId`
+ * (a pointer the owner has since moved to another storage is left alone).
+ */
+export declare function releaseOwnerName(redis: {
+    hget(key: string, field: string): Promise<string | null>;
+    hdel(key: string, ...fields: string[]): Promise<number>;
+}, indexKey: string, userId: string | null | undefined, name: string | null | undefined, storageId: string): Promise<void>;

package/dist/storage/utils.js

@@ -4,6 +4,9 @@ exports.uniqueKeyToRequestId = uniqueKeyToRequestId;
 exports.generateId = generateId;
 exports.nowISO = nowISO;
 exports.pathSegment = pathSegment;
+exports.ownerNameField = ownerNameField;
+exports.requireStorageOwner = requireStorageOwner;
+exports.releaseOwnerName = releaseOwnerName;
 const node_crypto_1 = require("node:crypto");
 function uniqueKeyToRequestId(uniqueKey) {
     return (0, node_crypto_1.createHash)('sha256')
@@ -34,3 +37,39 @@ function pathSegment(value) {
     }
     return encodeURIComponent(value);
 }
+/**
+ * Field of a named storage in the global `{prefix}datasets:index` /
+ * `kvs:index` / `rq:index` hash: scoped by owner (`name:{userId}:{name}`, the
+ * same field the server writes), so two accounts using one name never share
+ * or overwrite each other's pointer.
+ */
+function ownerNameField(userId, name) {
+    return `name:${userId}:${name}`;
+}
+/** The owner a new storage is created for: the active billing context's user. */
+function requireStorageOwner(billingContext, kind) {
+    const userId = billingContext?.userId;
+    if (!userId)
+        throw new Error(`Cannot create a ${kind} without a billing context (no owner user id).`);
+    return userId;
+}
+/**
+ * Remove an owner's name pointer, but only while it still points at `storageId`
+ * (a pointer the owner has since moved to another storage is left alone).
+ */
+async function releaseOwnerName(redis, indexKey, userId, name, storageId) {
+    if (!userId || !name)
+        return;
+    const field = ownerNameField(userId, name);
+    const raw = await redis.hget(indexKey, field);
+    if (!raw)
+        return;
+    try {
+        if (JSON.parse(raw)?.id !== storageId)
+            return;
+    }
+    catch {
+        return;
+    }
+    await redis.hdel(indexKey, field);
+}

package/dist/utils.d.ts

@@ -26,6 +26,8 @@ export interface EnqueueLinksOptions {
         url: string;
         userData?: Record<string, any>;
     } | null;
+    /** Most requests to enqueue from this page. Default: no limit */
+    limit?: number;
 }
 /**
  * Enqueues links found on the page using a CSS selector.

package/dist/utils.js

@@ -4,6 +4,8 @@ exports.utils = void 0;
 exports.enqueueLinks = enqueueLinks;
 exports.requestAsBrowser = requestAsBrowser;
 exports.getRandomUserAgent = getRandomUserAgent;
+/** Requests sent to the queue per addRequests call. */
+const ENQUEUE_BATCH_SIZE = 100;
 /**
  * Enqueues links found on the page using a CSS selector.
  *
@@ -22,7 +24,7 @@ exports.getRandomUserAgent = getRandomUserAgent;
  * ```
  */
 async function enqueueLinks(options) {
-    const { $, requestQueue, selector = 'a[href]', baseUrl, userData = {}, transformRequestFunction } = options;
+    const { $, requestQueue, selector = 'a[href]', baseUrl, userData = {}, transformRequestFunction, limit = Infinity, } = options;
     // Collect all matching URLs
     const urls = [];
     $(selector).each((_, element) => {
@@ -41,8 +43,11 @@ async function enqueueLinks(options) {
     });
     // Deduplicate URLs
     const uniqueUrls = [...new Set(urls)];
-    // Add each URL to the request queue
+    // Build the requests (up to `limit`), then add them to the queue in batches
+    const requests = [];
     for (const url of uniqueUrls) {
+        if (requests.length >= limit)
+            break;
         const request = {
             url,
             userData: { ...userData },
@@ -51,9 +56,11 @@ async function enqueueLinks(options) {
         const transformed = transformRequestFunction
             ? transformRequestFunction(request)
             : request;
-        if (transformed) {
-            await requestQueue.addRequest(transformed);
-        }
+        if (transformed)
+            requests.push(transformed);
+    }
+    for (let i = 0; i < requests.length; i += ENQUEUE_BATCH_SIZE) {
+        await requestQueue.addRequests(requests.slice(i, i + ENQUEUE_BATCH_SIZE));
     }
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.40",
+    "version": "1.2.41",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -44,6 +44,7 @@
         "test:charge-multi-event": "npx tsx --test tests/charge-multi-event.test.ts",
         "test:charge-multi-event-void": "npx tsx --test tests/charge-multi-event-void.test.ts",
         "test:charge-edge-cases": "npx tsx --test tests/charge-edge-cases.test.ts",
+        "test:charge-atomic-refusal-message": "npx tsx --test tests/charge-atomic-refusal-message.test.ts",
         "test:charge-pending-persist": "npx tsx --test tests/charge-pending-persist.test.ts",
         "test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
         "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",