scrapely 1.2.37 → 1.2.39

package/CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [1.2.39] - 2026-10-29
+
+- A refused charge names the limit that refused it. When a charge is refused or cut short, the platform now says why (HTTP 402 `error.data.reason`, and `limitedBy` on a partial acceptance): the run's item limit (`maxItems`), its maximum total charge (`maxTotalChargeUsd`) or the account balance. The SDK used to call all of them the spending limit. Now its log lines name the real limit; `ChargingManager.getChargeLimitReason()` returns `'max-items'`, `'max-charge'` or `'insufficient-funds'` and `getChargeLimitMessage()` the stop message ("Stopped: reached the item limit you set (800 items)", "Stopped: reached the maximum charge you set ($3.00)", "Stopped: your account has no credit left for this run"); and once a limit was reached, a status message (`Actor.setStatusMessage`, `Actor.exit`, `Actor.fail`) whose clause blames the maximum charge gets that clause replaced by the real stop message. Platforms that do not send the reason yet: inferred from the 402 message and `ACTOR_MAX_PAID_DATASET_ITEMS`. What is charged and saved does not change.
+
+## [1.2.38] - 2026-10-27
+
+- A key-value store record write (`Actor.setValue`, `KeyValueStore.setValue`) is no longer lost to one failed answer. Each write now sends an `Idempotency-Key`, and when its answer is lost (a `502`/`504` page from the edge or load balancer, a dropped connection, a timeout) it is sent again with the same key; the API applies and counts it once. Before, the write was never re-sent, so one such page on the final `OUTPUT` write failed the whole run. An API that answers a keyed write without `X-Scrapely-Idempotency` (an older server) gets no re-sends, as before.
+
 ## [1.2.37] - 2026-10-26
 
 - Opening a named dataset, key-value store or request queue (`Actor.openDataset('name')` and the like) retries when the API answers `503 storage-name-busy`. The API sends it when another request kept creating or renaming a storage of that name for over 15 s; nothing was changed, so the open is simply repeated. It used to fail the open.

package/dist/actor.js

@@ -1157,6 +1157,9 @@ class Actor {
     static async setStatusMessage(message, options) {
         const { isStatusMessageTerminal, level } = options || {};
         Actor._ensureActorInit('setStatusMessage');
+        // Actors word a refused charge as "the maximum charge for this run was
+        // reached" whatever refused it; name the limit that really stopped the run.
+        message = (0, charging_js_2.truthfulStatusMessage)(message);
         const loggedStatusMessage = `[Status message]: ${message}`;
         switch (level) {
             case 'DEBUG':

package/dist/charging.d.ts

@@ -21,14 +21,40 @@ export declare class ChargeRejectedError extends Error {
     readonly count: number;
     readonly statusCode?: number;
     readonly reason: string;
+    /** Which run limit refused the charge, when the platform said (see ChargeLimitReason). */
+    readonly limitReason?: ChargeLimitReason;
+    readonly maxItems?: number;
     constructor(message: string, opts: {
         operationId: string;
         eventName: string;
         count: number;
         statusCode?: number;
         reason: string;
+        limitReason?: ChargeLimitReason;
+        maxItems?: number;
     });
 }
+/**
+ * Which run limit refused or cut down a charge: the platform's `error.reason`
+ * on HTTP 402 and `limitedBy` on a partial acceptance. 'max-items' is the item
+ * limit the user set (maxItems), 'max-charge' the maximum total charge
+ * (maxTotalChargeUsd), 'insufficient-funds' the account balance. The run's
+ * stop message names it, so an item limit is never called a spending limit.
+ */
+export type ChargeLimitReason = 'max-items' | 'max-charge' | 'insufficient-funds';
+/** The run's stop message for a charge limit, or undefined when the reason is unknown. */
+export declare function chargeLimitStopMessage(reason: ChargeLimitReason | undefined, opts?: {
+    maxItems?: number | null;
+    maxTotalChargeUsd?: number | null;
+}): string | undefined;
+/**
+ * Replace the clause of `message` that blames the spending limit with
+ * `stopMessage`: "Saved 800 results; stopped because the maximum charge for
+ * this run was reached" becomes "Saved 800 results. Stopped: reached the item
+ * limit you set (800 items)". Only the first such clause is replaced (later
+ * ones are dropped); a message without one is returned as is.
+ */
+export declare function rewriteChargeLimitClaims(message: string, stopMessage: string | undefined): string;
 /** @internal Register the manager whose limit guards default-dataset writes. */
 export declare function setActiveChargingManager(manager: ChargingManager | undefined): void;
 /** @internal The run's default dataset ID, as the platform passes it. */
@@ -39,6 +65,8 @@ export declare function defaultDatasetIdFromEnv(): string | undefined;
  * clients for writes that did not go through `pushDataAndCharge`
  * (`Actor.openDataset().pushData()`, crawler `pushData`, ...).
  */
+/** `message` naming the limit that really stopped the run (Actor.setStatusMessage). */
+export declare function truthfulStatusMessage(message: string): string;
 export declare function refuseDefaultDatasetPush(itemCount: number): boolean;
 export declare const APIFY_DEFAULT_DATASET_ITEM_EVENT = "apify-default-dataset-item";
 /** Scrapely-native synthetic event for default-dataset items. */
@@ -74,6 +102,12 @@ export declare class ChargingManager {
     private _closed;
     /** Sticky: the platform refused or capped a charge (HTTP 402 / fewer accepted) or the local gate trimmed one. */
     private chargeLimitHit;
+    /** Which limit that was (the first one known wins) and the item limit the platform reported. */
+    private chargeLimitReason;
+    private chargeLimitMaxItems;
+    /** limitedBy / maxItems of the last partially accepted charge response. */
+    private lastLimitedBy;
+    private lastLimitMaxItems;
     /** Default-dataset rows refused after the limit (one warning, then a count). */
     private refusedDefaultDatasetItems;
     /**
@@ -124,10 +158,33 @@ export declare class ChargingManager {
      */
     isChargeLimitReached(): boolean;
     private markChargeLimitHit;
+    private markRejectedCharge;
+    /** The platform accepted fewer events than asked: record which limit cut it. */
+    private markPartialCharge;
+    /** The run's item limit: as the platform reported it, else ACTOR_MAX_PAID_DATASET_ITEMS. */
+    private maxItemsSetting;
+    /**
+     * Why this run's charges stopped being accepted, or undefined while they
+     * still are: 'max-items' (the item limit the user set), 'max-charge' (the
+     * maximum total charge) or 'insufficient-funds' (the account balance).
+     * Platforms that do not say which limit it was: inferred from the run's settings.
+     */
+    getChargeLimitReason(): ChargeLimitReason | undefined;
+    /**
+     * The run's stop message for the limit that was reached, or undefined:
+     * "Stopped: reached the item limit you set (800 items)", "Stopped: reached
+     * the maximum charge you set ($3.00)", ... Use it as (part of) the final
+     * status message when the Actor stops because a charge was refused.
+     */
+    getChargeLimitMessage(): string | undefined;
+    /** `message` with a clause blaming the spending limit replaced by the real reason (once a limit was reached). */
+    rewriteStatusMessage(message: string): string;
+    /** 'the item limit of this run (maxItems = 800)', ... for log lines. */
+    private limitName;
     /** @internal Count rows dropped because of the limit; warn on the first one only. */
     _noteRefusedDefaultDatasetItems(count: number): void;
-    /** @internal */
-    _markChargeLimitHit(): void;
+    /** @internal `reason`: the limit that trimmed the push, when known. */
+    _markChargeLimitHit(reason?: ChargeLimitReason): void;
     /** @internal Whether every row of a push with these events is charged a positive price. */
     _isPaidPerItem(eventName: string | undefined, isDefaultDataset: boolean): boolean;
     /** @internal */

package/dist/charging.js

@@ -1,8 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ChargingManager = exports.DEFAULT_DATASET_ITEM_EVENT = exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = exports.APIFY_DEFAULT_DATASET_ITEM_EVENT = exports.ChargeRejectedError = exports.ChargeUncertainError = exports.CHARGE_IDEMPOTENCY_SAFE_AGE_MS = void 0;
+exports.chargeLimitStopMessage = chargeLimitStopMessage;
+exports.rewriteChargeLimitClaims = rewriteChargeLimitClaims;
 exports.setActiveChargingManager = setActiveChargingManager;
 exports.defaultDatasetIdFromEnv = defaultDatasetIdFromEnv;
+exports.truthfulStatusMessage = truthfulStatusMessage;
 exports.refuseDefaultDatasetPush = refuseDefaultDatasetPush;
 exports.isDefaultDatasetItemEvent = isDefaultDatasetItemEvent;
 exports.resolveDefaultDatasetItemEventName = resolveDefaultDatasetItemEventName;
@@ -57,6 +60,9 @@ class ChargeRejectedError extends Error {
     count;
     statusCode;
     reason;
+    /** Which run limit refused the charge, when the platform said (see ChargeLimitReason). */
+    limitReason;
+    maxItems;
     constructor(message, opts) {
         super(message);
         this.name = 'ChargeRejectedError';
@@ -65,9 +71,99 @@ class ChargeRejectedError extends Error {
         this.count = opts.count;
         this.statusCode = opts.statusCode;
         this.reason = opts.reason;
+        this.limitReason = opts.limitReason;
+        this.maxItems = opts.maxItems;
     }
 }
 exports.ChargeRejectedError = ChargeRejectedError;
+const CHARGE_LIMIT_REASONS = new Set(['max-items', 'max-charge', 'insufficient-funds']);
+function asChargeLimitReason(raw) {
+    return typeof raw === 'string' && CHARGE_LIMIT_REASONS.has(raw) ? raw : undefined;
+}
+function positiveInt(raw) {
+    return typeof raw === 'number' && Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : undefined;
+}
+/** $3.00, $0.005: two decimals unless the amount needs more. */
+function formatUsd(usd) {
+    return Math.abs(usd * 100 - Math.round(usd * 100)) < 1e-9 ? `$${usd.toFixed(2)}` : `$${Number(usd.toPrecision(6))}`;
+}
+/** The run's stop message for a charge limit, or undefined when the reason is unknown. */
+function chargeLimitStopMessage(reason, opts = {}) {
+    if (reason === 'max-items') {
+        const n = opts.maxItems;
+        return n && n > 0
+            ? `Stopped: reached the item limit you set (${n} ${n === 1 ? 'item' : 'items'})`
+            : 'Stopped: reached the item limit you set';
+    }
+    if (reason === 'max-charge') {
+        const usd = opts.maxTotalChargeUsd;
+        return typeof usd === 'number' && Number.isFinite(usd)
+            ? `Stopped: reached the maximum charge you set (${formatUsd(usd)})`
+            : 'Stopped: reached the maximum charge you set';
+    }
+    if (reason === 'insufficient-funds')
+        return 'Stopped: your account has no credit left for this run';
+    return undefined;
+}
+// A status-message clause that blames the spending limit ("stopped because the
+// maximum charge for this run was reached", "The run reached its maximum total
+// charge. Stopping.", ...). Actors written before the platform said which limit
+// was hit use such texts for every refused charge.
+const CHARGE_CLAIM_RE = /\bmax(?:imum)?\s+(?:total\s+)?charge|\bspending\s+limit/i;
+const CLAUSE_SEPARATOR_RE = /(;\s*|[.!?](?:\s+|$))/;
+/**
+ * Replace the clause of `message` that blames the spending limit with
+ * `stopMessage`: "Saved 800 results; stopped because the maximum charge for
+ * this run was reached" becomes "Saved 800 results. Stopped: reached the item
+ * limit you set (800 items)". Only the first such clause is replaced (later
+ * ones are dropped); a message without one is returned as is.
+ */
+function rewriteChargeLimitClaims(message, stopMessage) {
+    if (!message || !stopMessage || message.includes(stopMessage) || !CHARGE_CLAIM_RE.test(message))
+        return message;
+    const parts = message.split(CLAUSE_SEPARATOR_RE);
+    const out = [];
+    let replaced = false;
+    for (let i = 0; i < parts.length; i += 2) {
+        const clause = parts[i] ?? '';
+        const sep = parts[i + 1] ?? '';
+        if (!CHARGE_CLAIM_RE.test(clause)) {
+            out.push(clause + sep);
+            continue;
+        }
+        if (replaced)
+            continue;
+        replaced = true;
+        const prev = out.length ? out[out.length - 1].trimEnd() : '';
+        if (prev.endsWith(';'))
+            out[out.length - 1] = `${prev.slice(0, -1).trimEnd()}. `;
+        const lead = clause.slice(0, clause.length - clause.trimStart().length);
+        let tail = '';
+        if (sep)
+            tail = [';', '.', '!', '?'].includes(sep.trim()) ? `.${sep.slice(sep.trimEnd().length)}` : sep;
+        out.push(lead + stopMessage + tail);
+    }
+    let result = out.join('').trimEnd();
+    if (message.trimEnd().endsWith('.') && !result.endsWith('.'))
+        result += '.';
+    return result;
+}
+/** Which limit a 402 names: error.reason / error.data.reason, else its message (older platforms). */
+function rejectionLimit(err, message) {
+    const e = err;
+    const data = e?.data && typeof e.data === 'object' ? e.data : undefined;
+    let limitReason = asChargeLimitReason(data?.reason) ?? asChargeLimitReason(e?.body?.error?.reason);
+    const maxItems = positiveInt(data?.maxItems) ?? positiveInt(e?.body?.maxItems);
+    if (!limitReason) {
+        if (/maxitems|maximum number of results/i.test(message))
+            limitReason = 'max-items';
+        else if (/insufficient balance/i.test(message))
+            limitReason = 'insufficient-funds';
+        else if (/maximum total charge/i.test(message))
+            limitReason = 'max-charge';
+    }
+    return { limitReason, maxItems };
+}
 /** True when a ChargeRejectedError means the run's spending limit is used up. */
 function isChargeLimitRejection(err) {
     return (err.statusCode === 402 ||
@@ -101,6 +197,10 @@ function defaultDatasetIdFromEnv() {
  * clients for writes that did not go through `pushDataAndCharge`
  * (`Actor.openDataset().pushData()`, crawler `pushData`, ...).
  */
+/** `message` naming the limit that really stopped the run (Actor.setStatusMessage). */
+function truthfulStatusMessage(message) {
+    return activeChargingManager ? activeChargingManager.rewriteStatusMessage(message) : message;
+}
 function refuseDefaultDatasetPush(itemCount) {
     if (chargeGatedPush.getStore())
         return false;
@@ -154,6 +254,12 @@ class ChargingManager {
     _closed = false;
     /** Sticky: the platform refused or capped a charge (HTTP 402 / fewer accepted) or the local gate trimmed one. */
     chargeLimitHit = false;
+    /** Which limit that was (the first one known wins) and the item limit the platform reported. */
+    chargeLimitReason;
+    chargeLimitMaxItems;
+    /** limitedBy / maxItems of the last partially accepted charge response. */
+    lastLimitedBy;
+    lastLimitMaxItems;
     /** Default-dataset rows refused after the limit (one warning, then a count). */
     refusedDefaultDatasetItems = 0;
     /**
@@ -214,7 +320,7 @@ class ChargingManager {
     close() {
         this._closed = true;
         if (this.refusedDefaultDatasetItems > 0) {
-            console.warn(`[Scrapely] ${this.refusedDefaultDatasetItems} result(s) pushed to the default dataset after the spending limit was reached were not saved`);
+            console.warn(`[Scrapely] ${this.refusedDefaultDatasetItems} result(s) pushed to the default dataset after ${this.limitName()} was reached were not saved`);
         }
     }
     /**
@@ -242,25 +348,104 @@ class ChargingManager {
         // Same rounding as the budget gate, so 0.03 - 0.02 still pays a $0.01 event.
         return this.calculateMaxChargesByPrice(Math.min(...prices)) <= 0;
     }
-    markChargeLimitHit() {
-        if (this.isPayPerEvent)
-            this.chargeLimitHit = true;
+    markChargeLimitHit(reason, maxItems) {
+        if (!this.isPayPerEvent)
+            return;
+        this.chargeLimitHit = true;
+        if (reason && this.chargeLimitReason === undefined)
+            this.chargeLimitReason = reason;
+        if (reason === 'max-items' && maxItems !== undefined && maxItems > 0)
+            this.chargeLimitMaxItems = maxItems;
+    }
+    markRejectedCharge(err) {
+        if (isChargeLimitRejection(err))
+            this.markChargeLimitHit(err.limitReason, err.maxItems);
+    }
+    /** The platform accepted fewer events than asked: record which limit cut it. */
+    markPartialCharge() {
+        this.markChargeLimitHit(this.lastLimitedBy, this.lastLimitMaxItems);
+    }
+    /** The run's item limit: as the platform reported it, else ACTOR_MAX_PAID_DATASET_ITEMS. */
+    maxItemsSetting() {
+        if (this.chargeLimitMaxItems !== undefined)
+            return this.chargeLimitMaxItems;
+        const raw = Number(process.env.ACTOR_MAX_PAID_DATASET_ITEMS ?? process.env.APIFY_ACTOR_MAX_PAID_DATASET_ITEMS);
+        return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : undefined;
+    }
+    /**
+     * Why this run's charges stopped being accepted, or undefined while they
+     * still are: 'max-items' (the item limit the user set), 'max-charge' (the
+     * maximum total charge) or 'insufficient-funds' (the account balance).
+     * Platforms that do not say which limit it was: inferred from the run's settings.
+     */
+    getChargeLimitReason() {
+        if (!this.isChargeLimitReached())
+            return undefined;
+        if (this.chargeLimitReason)
+            return this.chargeLimitReason;
+        const maxItems = this.maxItemsSetting();
+        if (maxItems !== undefined &&
+            Object.entries(this.chargingState ?? {}).some(([name, state]) => !isSyntheticEvent(name) && (this.pricingInfo[name]?.price ?? 0) > 0 && state.chargeCount >= maxItems)) {
+            return 'max-items';
+        }
+        if (Number.isFinite(this.maxTotalChargeUsd)) {
+            const prices = Object.values(this.pricingInfo).map(({ price }) => price).filter((price) => price > 0);
+            if (prices.length > 0 && this.calculateMaxChargesByPrice(Math.min(...prices)) <= 0)
+                return 'max-charge';
+        }
+        return undefined;
+    }
+    /**
+     * The run's stop message for the limit that was reached, or undefined:
+     * "Stopped: reached the item limit you set (800 items)", "Stopped: reached
+     * the maximum charge you set ($3.00)", ... Use it as (part of) the final
+     * status message when the Actor stops because a charge was refused.
+     */
+    getChargeLimitMessage() {
+        return chargeLimitStopMessage(this.getChargeLimitReason(), {
+            maxItems: this.maxItemsSetting(),
+            maxTotalChargeUsd: Number.isFinite(this.maxTotalChargeUsd) ? this.maxTotalChargeUsd : null,
+        });
+    }
+    /** `message` with a clause blaming the spending limit replaced by the real reason (once a limit was reached). */
+    rewriteStatusMessage(message) {
+        try {
+            return rewriteChargeLimitClaims(message, this.getChargeLimitMessage());
+        }
+        catch {
+            return message;
+        }
+    }
+    /** 'the item limit of this run (maxItems = 800)', ... for log lines. */
+    limitName() {
+        const reason = this.getChargeLimitReason();
+        if (reason === 'max-items') {
+            const n = this.maxItemsSetting();
+            return n ? `the item limit of this run (maxItems = ${n})` : 'the item limit of this run (maxItems)';
+        }
+        if (reason === 'insufficient-funds')
+            return 'the end of the account balance';
+        if (reason === 'max-charge' || Number.isFinite(this.maxTotalChargeUsd)) {
+            const limit = Number.isFinite(this.maxTotalChargeUsd) ? `$${this.maxTotalChargeUsd}` : 'the limit';
+            return `the spending limit of this run (maxTotalChargeUsd = ${limit})`;
+        }
+        return 'the limit of this run';
     }
     /** @internal Count rows dropped because of the limit; warn on the first one only. */
     _noteRefusedDefaultDatasetItems(count) {
         if (count <= 0)
             return;
         if (this.refusedDefaultDatasetItems === 0) {
-            const limit = Number.isFinite(this.maxTotalChargeUsd) ? `$${this.maxTotalChargeUsd}` : 'the limit';
-            console.warn(`[Scrapely] The spending limit of this run (maxTotalChargeUsd = ${limit}) has been reached: ` +
+            const name = this.limitName();
+            console.warn(`[Scrapely] ${name[0].toUpperCase()}${name.slice(1)} has been reached: ` +
                 'results pushed to the default dataset are no longer saved. The Actor should stop now ' +
                 '(check ChargeResult.eventChargeLimitReached or Actor.getChargingManager().isChargeLimitReached()).');
         }
         this.refusedDefaultDatasetItems += count;
     }
-    /** @internal */
-    _markChargeLimitHit() {
-        this.markChargeLimitHit();
+    /** @internal `reason`: the limit that trimmed the push, when known. */
+    _markChargeLimitHit(reason) {
+        this.markChargeLimitHit(reason);
     }
     /** @internal Whether every row of a push with these events is charged a positive price. */
     _isPaidPerItem(eventName, isDefaultDataset) {
@@ -728,6 +913,8 @@ class ChargingManager {
                 const accepted = readAcceptedCount(body, intent.count);
                 const remaining = body.remainingBudgetNd;
                 this.lastRemainingBudgetNd = typeof remaining === 'number' && Number.isFinite(remaining) && remaining >= 0 ? remaining : null;
+                this.lastLimitedBy = asChargeLimitReason(body.limitedBy);
+                this.lastLimitMaxItems = positiveInt(body.maxItems);
                 return accepted;
             }
             catch (err) {
@@ -966,7 +1153,7 @@ class ChargingManager {
                         const accepted = await this.postChargeWithRetries(intent, deadline);
                         if (accepted !== null) {
                             if (accepted < intent.count)
-                                this.markChargeLimitHit();
+                                this.markPartialCharge();
                             const eventPrice = this.getEventPricing(intent.eventName).price;
                             this.applyAcceptedCount(intent.eventName, intent.count, accepted, eventPrice);
                             this.pendingCharges.delete(intent.idempotencyKey);
@@ -975,8 +1162,7 @@ class ChargingManager {
                     }
                     catch (err) {
                         if (err instanceof ChargeRejectedError) {
-                            if (isChargeLimitRejection(err))
-                                this.markChargeLimitHit();
+                            this.markRejectedCharge(err);
                             const eventPrice = this.getEventPricing(intent.eventName).price;
                             this.applyAcceptedCount(intent.eventName, intent.count, 0, eventPrice);
                             this.pendingCharges.delete(intent.idempotencyKey);
@@ -1049,8 +1235,9 @@ class ChargingManager {
             }
             return 0;
         })();
+        // The local budget gate: only maxTotalChargeUsd trims here.
         if (chargedCount < count)
-            this.markChargeLimitHit();
+            this.markChargeLimitHit('max-charge');
         if (chargedCount === 0) {
             return {
                 eventChargeLimitReached: count > 0,
@@ -1093,8 +1280,7 @@ class ChargingManager {
             }
             catch (err) {
                 if (err instanceof ChargeRejectedError) {
-                    if (isChargeLimitRejection(err))
-                        this.markChargeLimitHit();
+                    this.markRejectedCharge(err);
                     // Server definitively rejected: unwind local reservation.
                     this.pendingCharges.delete(intent.idempotencyKey);
                     chargedCount = this.applyAcceptedCount(normalizedEventName, chargedCount, 0, eventPricing.price);
@@ -1112,7 +1298,7 @@ class ChargingManager {
                     this.lastAcceptedOperationId = intent.idempotencyKey;
                 // The platform capped the charge at the run's limit.
                 if (accepted < chargedCount)
-                    this.markChargeLimitHit();
+                    this.markPartialCharge();
                 chargedCount = this.applyAcceptedCount(normalizedEventName, chargedCount, accepted, eventPricing.price);
                 this.syncPlatformBudget();
             }
@@ -1143,7 +1329,7 @@ class ChargingManager {
         }
         if (chargedCount < count) {
             const subject = count === 1 ? 'instance' : 'instances';
-            console.log(`[Scrapely] Charging ${count} ${subject} of '${normalizedEventName}' event would exceed maxTotalChargeUsd - only ${chargedCount} events were charged`);
+            console.log(`[Scrapely] Charging ${count} ${subject} of '${normalizedEventName}' event would exceed ${this.limitName()} - only ${chargedCount} events were charged`);
         }
         return {
             eventChargeLimitReached: this.calculateMaxEventChargeCountWithinLimit(normalizedEventName) <= 0,
@@ -1197,6 +1383,7 @@ function classifyChargeHttpError(err, intent) {
             count: intent.count,
             statusCode: status,
             reason: status === 402 ? 'charge-limit-exceeded' : `http-${status}`,
+            ...(status === 402 ? rejectionLimit(err, String(err?.message ?? '')) : {}),
         });
     }
     // A 5xx / 408 / 429 answer is never a definitive rejection, whatever its text
@@ -1213,6 +1400,7 @@ function classifyChargeHttpError(err, intent) {
             count: intent.count,
             statusCode: status,
             reason: 'message-match',
+            ...(/charge-limit-exceeded/i.test(message) ? rejectionLimit(err, message) : {}),
         });
     }
     return null;
@@ -1302,7 +1490,7 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
         const allItemsTrimmed = itemsArray.length > 0 && limitedItems.length === 0;
         if (isDefaultDataset && limitedItems.length < itemsArray.length) {
             // Rows beyond what the budget can pay for are dropped.
-            chargingManager._markChargeLimitHit?.();
+            chargingManager._markChargeLimitHit?.('max-charge');
             chargingManager._noteRefusedDefaultDatasetItems?.(itemsArray.length - limitedItems.length);
         }
         if (limitedItems.length === 0) {

package/dist/events/events-client.js

@@ -31,8 +31,9 @@ function connectEvents(eventsWsUrl, token, runId) {
         return;
     shouldReconnect = true;
     try {
-        const url = `${eventsWsUrl}?token=${encodeURIComponent(token)}&runId=${encodeURIComponent(runId)}`;
-        ws = new ws_1.default(url);
+        // The token goes in a header, not the URL: URLs end up in access logs.
+        const url = `${eventsWsUrl}?runId=${encodeURIComponent(runId)}`;
+        ws = new ws_1.default(url, { headers: { Authorization: `Bearer ${token}` } });
         ws.onopen = () => {
             connected = true;
             console.log(`[Scrapely Events] Connected to ${eventsWsUrl}`);

package/dist/storage/api-storage-client.d.ts

@@ -26,7 +26,14 @@ export declare class ApiStorageClientError extends Error {
 export declare function isRetryableResponse(status: number, idempotent: boolean, errorType?: string, bodyText?: string): boolean;
 /** Default time budget for retrying one API call (429, 503, network errors). */
 export declare const DEFAULT_API_RETRY_WINDOW_MS = 300000;
-/** Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes. */
+/** A key-value store record URL (`.../key-value-stores/<id>/records/<key>`). */
+export declare function isKvRecordUrl(url: string): boolean;
+/**
+ * Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes.
+ * Record PUTs count as de-duplicated until a keyed record PUT was stored without the support
+ * header: re-applying the same body to the same record leaves the same record, so the worst
+ * case on an API that ignores the key is one extra counted write, while not re-sending fails the run.
+ */
 export declare function apiDedupesKeyedWrites(url?: string): boolean;
 /** Test hook: forget what was learned about the API, or force an answer for every origin. */
 export declare function _resetKeyedWriteSupport(value?: boolean): void;

package/dist/storage/api-storage-client.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.BaseApiClient = exports.DEFAULT_API_RETRY_WINDOW_MS = exports.ApiStorageClientError = void 0;
 exports.isRetryableResponse = isRetryableResponse;
+exports.isKvRecordUrl = isKvRecordUrl;
 exports.apiDedupesKeyedWrites = apiDedupesKeyedWrites;
 exports._resetKeyedWriteSupport = _resetKeyedWriteSupport;
 exports.apiRetryWindowMs = apiRetryWindowMs;
@@ -139,7 +140,9 @@ function isApiStorageClientError(err) {
     return err instanceof ApiStorageClientError;
 }
 // PUT is left out: a KV record PUT is counted (and billed) as a write each
-// time it is applied, so a lost response must not be re-sent blindly.
+// time it is applied, so a lost response must not be re-sent blindly. Record
+// PUTs carry an Idempotency-Key instead (see isKvRecordUrl), which the API
+// de-duplicates, so they are re-sent after a lost answer (e.g. an edge 502).
 const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'DELETE', 'OPTIONS']);
 // HAProxy's own 503 page: no backend was available, so nothing was forwarded.
 const NO_BACKEND_MARKER = 'No server is available to handle this request';
@@ -180,20 +183,39 @@ const IDEMPOTENCY_SUPPORT_HEADER = 'x-scrapely-idempotency';
 const keyedWritesDeduped = new Map();
 /** Test override for every origin (undefined: learn from answers). */
 let keyedWritesDedupedOverride;
-function originOf(url) {
+/** A key-value store record URL (`.../key-value-stores/<id>/records/<key>`). */
+function isKvRecordUrl(url) {
+    let path = url;
     try {
-        return new URL(url).origin;
+        path = new URL(url).pathname;
     }
-    catch {
-        return url;
+    catch { /* a path already */ }
+    return path.includes('/key-value-stores/') && path.includes('/records/');
+}
+/** What is learned per API: its origin, separately for record PUTs and other keyed writes. */
+function originOf(url) {
+    let origin = url;
+    try {
+        origin = new URL(url).origin;
     }
+    catch { /* keep the input */ }
+    return isKvRecordUrl(url) ? `${origin}#kv-record` : origin;
 }
-/** Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes. */
+/**
+ * Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes.
+ * Record PUTs count as de-duplicated until a keyed record PUT was stored without the support
+ * header: re-applying the same body to the same record leaves the same record, so the worst
+ * case on an API that ignores the key is one extra counted write, while not re-sending fails the run.
+ */
 function apiDedupesKeyedWrites(url) {
     if (keyedWritesDedupedOverride !== undefined)
         return keyedWritesDedupedOverride;
-    if (url !== undefined)
-        return keyedWritesDeduped.get(originOf(url)) === true;
+    if (url !== undefined) {
+        const learned = keyedWritesDeduped.get(originOf(url));
+        if (learned === undefined && isKvRecordUrl(url))
+            return true;
+        return learned === true;
+    }
     return [...keyedWritesDeduped.values()].pop() === true;
 }
 /** Test hook: forget what was learned about the API, or force an answer for every origin. */
@@ -792,10 +814,13 @@ class ApiKeyValueStoreClient extends BaseApiClient {
             ct = 'application/json';
         }
         // Pass timeout/retry options from KeyValueStoreRecordOptions to the API request
-        const requestOptions = options ? {
-            timeoutSecs: options.timeoutSecs,
-            doNotRetryTimeouts: options.doNotRetryTimeouts,
-        } : undefined;
+        // One Idempotency-Key for every attempt of this write: a re-send after a lost answer
+        // (edge 502/504, dropped connection) is applied and counted once.
+        const requestOptions = {
+            timeoutSecs: options?.timeoutSecs,
+            doNotRetryTimeouts: options?.doNotRetryTimeouts,
+            idempotencyKey: (0, node_crypto_1.randomUUID)(),
+        };
         await this.request('PUT', `/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`, body, contentType || ct, requestOptions);
     }
     async deleteRecord(key) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.37",
+    "version": "1.2.39",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -75,7 +75,8 @@
         "test:charge-retries": "npx tsx --test tests/charge-retries.test.ts",
         "test:solver": "npx tsx --test tests/solver.test.ts",
         "test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts",
-        "test:v2-utils": "npx tsx --test tests/v2-utils.test.ts"
+        "test:v2-utils": "npx tsx --test tests/v2-utils.test.ts",
+        "test:default-api-url": "npx tsx --test tests/default-api-url.test.ts"
     },
     "author": "TeraShift GmbH (Scrape.ly)",
     "license": "ISC AND Apache-2.0",
@@ -92,7 +93,7 @@
         "lodash": "^4.18.1",
         "mysql2": "^3.23.1",
         "playwright": "^1.58.2",
-        "scrapely-client": "^2.25.2",
+        "scrapely-client": "^2.25.9",
         "tar-fs": "2.1.5",
         "ws": "^8.18.0"
     },
@@ -130,6 +131,7 @@
         "@inquirer/external-editor": "3.0.3",
         "nanoid": "5.1.16",
         "browserslist": "4.28.7",
-        "basic-ftp": "^6.2.3"
+        "basic-ftp": "^6.2.3",
+        "@asamuzakjp/css-color": "^7.0.1"
     }
 }