scrapely 1.2.14 → 1.2.16

package/dist/actor.js

@@ -273,6 +273,8 @@ class Actor {
             apifyClientInstance = this._apifyClient;
             apifyClientAssigned = true;
             await this.chargingManager.init();
+            // Storage clients refuse default-dataset writes past this run's limit.
+            (0, charging_js_1.setActiveChargingManager)(this.chargingManager);
             // Flush pending PPE intents before hard exit so uncertain charges can still settle.
             const gracefulSignalExit = (signal) => {
                 const doExit = () => {
@@ -517,6 +519,14 @@ class Actor {
         const { Dataset } = await import('crawlee');
         const dataset = await Dataset.open();
         if (eventName === undefined && !this.usesPushDataInterception(dataset)) {
+            // Charge-limit safety net (also for local storage): this push is not
+            // charged, so past the run's spending limit it is not stored.
+            if (this.initialized &&
+                dataset.id === this.config.get('defaultDatasetId') &&
+                this.chargingManager.isChargeLimitReached()) {
+                this.chargingManager._noteRefusedDefaultDatasetItems(Array.isArray(item) ? item.length : 1);
+                return;
+            }
             await dataset.pushData(item);
             return;
         }
@@ -951,6 +961,7 @@ class Actor {
         await eventManager.close();
         (0, events_client_js_1.disconnectEvents)();
         (0, patch_storage_js_1.disconnectStorage)();
+        (0, charging_js_1.setActiveChargingManager)(undefined);
         // Let buffered stdout/stderr reach the log pipe before the process exits
         // (pipe writes are async in Node). Storage was flushed above, so this
         // replaces the old fixed 1 s sleep: it resolves as soon as both streams

package/dist/charging.d.ts

@@ -29,6 +29,17 @@ export declare class ChargeRejectedError extends Error {
         reason: string;
     });
 }
+/** @internal Register the manager whose limit guards default-dataset writes. */
+export declare function setActiveChargingManager(manager: ChargingManager | undefined): void;
+/** @internal The run's default dataset ID, as the platform passes it. */
+export declare function defaultDatasetIdFromEnv(): string | undefined;
+/**
+ * @internal True when a write of `itemCount` rows to the default dataset must
+ * be dropped (pay-per-event run past its spending limit). Called by storage
+ * clients for writes that did not go through `pushDataAndCharge`
+ * (`Actor.openDataset().pushData()`, crawler `pushData`, ...).
+ */
+export declare function refuseDefaultDatasetPush(itemCount: number): boolean;
 export declare const APIFY_DEFAULT_DATASET_ITEM_EVENT = "apify-default-dataset-item";
 /** Scrapely-native synthetic event for default-dataset items. */
 export declare const SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = "scrapely-default-dataset-item";
@@ -62,8 +73,32 @@ export declare class ChargingManager {
     private readonly apifyClient;
     /** Set during actor shutdown; rejects new charge operations after the final flush. */
     private _closed;
+    /** Sticky: the platform refused or capped a charge (HTTP 402 / fewer accepted) or the local gate trimmed one. */
+    private chargeLimitHit;
+    /** Default-dataset rows refused after the limit (one warning, then a count). */
+    private refusedDefaultDatasetItems;
     /** @internal Called by Actor shutdown before the final flush. */
     close(): void;
+    /**
+     * Whether this pay-per-event run has used up its spending limit.
+     *
+     * True once a charge was refused or capped by the limit (the platform's
+     * HTTP 402 or a partial acceptance, or the local budget gate), or once the
+     * remaining budget cannot pay for one event of the cheapest priced type.
+     * From then on, pushes to the default dataset that are not paid for are
+     * refused. Always false for actors without pay-per-event pricing, without
+     * a limit, or with only free events.
+     */
+    isChargeLimitReached(): boolean;
+    private markChargeLimitHit;
+    /** @internal Count rows dropped because of the limit; warn on the first one only. */
+    _noteRefusedDefaultDatasetItems(count: number): void;
+    /** @internal */
+    _markChargeLimitHit(): void;
+    /** @internal Whether every row of a push with these events is charged a positive price. */
+    _isPaidPerItem(eventName: string | undefined, isDefaultDataset: boolean): boolean;
+    /** @internal */
+    _refuseDefaultDatasetPush(itemCount: number): boolean;
     /** @internal Whether shutdown has closed this manager to new charges. */
     isClosed(): boolean;
     private readonly LOCAL_CHARGING_LOG_DATASET_NAME;

package/dist/charging.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ChargingManager = exports.DEFAULT_DATASET_ITEM_EVENT = exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = exports.APIFY_DEFAULT_DATASET_ITEM_EVENT = exports.ChargeRejectedError = exports.ChargeUncertainError = exports.CHARGE_IDEMPOTENCY_SAFE_AGE_MS = void 0;
+exports.setActiveChargingManager = setActiveChargingManager;
+exports.defaultDatasetIdFromEnv = defaultDatasetIdFromEnv;
+exports.refuseDefaultDatasetPush = refuseDefaultDatasetPush;
 exports.isDefaultDatasetItemEvent = isDefaultDatasetItemEvent;
 exports.resolveDefaultDatasetItemEventName = resolveDefaultDatasetItemEventName;
 exports.hasDefaultDatasetItemEventInPricing = hasDefaultDatasetItemEventInPricing;
@@ -11,6 +14,7 @@ exports.pushCertainlyFailed = pushCertainlyFailed;
 // Derived from apify-sdk-js (src/charging.ts),
 // Copyright Apify Technologies s.r.o., licensed under the Apache License 2.0.
 // Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
+const node_async_hooks_1 = require("node:async_hooks");
 const node_crypto_1 = require("node:crypto");
 /** Client safe-replay window; server TTL is longer (35d). */
 exports.CHARGE_IDEMPOTENCY_SAFE_AGE_MS = 30 * 24 * 60 * 60 * 1000;
@@ -48,6 +52,44 @@ class ChargeRejectedError extends Error {
     }
 }
 exports.ChargeRejectedError = ChargeRejectedError;
+/** True when a ChargeRejectedError means the run's spending limit is used up. */
+function isChargeLimitRejection(err) {
+    return (err.statusCode === 402 ||
+        err.reason === 'charge-limit-exceeded' ||
+        /charge-limit-exceeded/i.test(err.message));
+}
+/*
+ * Charge-limit safety net for the default dataset.
+ *
+ * Once a pay-per-event run has used up its spending limit (maxTotalChargeUsd),
+ * rows written to the run's default dataset would go out unbilled. The SDK then
+ * refuses such writes: nothing is stored, one warning is logged and
+ * `ChargingManager.isChargeLimitReached()` turns true. Pushes made by
+ * `pushDataAndCharge` are decided there and run inside `chargeGatedPush`, so
+ * the storage client does not check them twice.
+ */
+const chargeGatedPush = new node_async_hooks_1.AsyncLocalStorage();
+/** The charging manager of the running Actor; storage clients consult it. */
+let activeChargingManager;
+/** @internal Register the manager whose limit guards default-dataset writes. */
+function setActiveChargingManager(manager) {
+    activeChargingManager = manager;
+}
+/** @internal The run's default dataset ID, as the platform passes it. */
+function defaultDatasetIdFromEnv() {
+    return process.env.ACTOR_DEFAULT_DATASET_ID || process.env.APIFY_DEFAULT_DATASET_ID || undefined;
+}
+/**
+ * @internal True when a write of `itemCount` rows to the default dataset must
+ * be dropped (pay-per-event run past its spending limit). Called by storage
+ * clients for writes that did not go through `pushDataAndCharge`
+ * (`Actor.openDataset().pushData()`, crawler `pushData`, ...).
+ */
+function refuseDefaultDatasetPush(itemCount) {
+    if (chargeGatedPush.getStore())
+        return false;
+    return activeChargingManager?._refuseDefaultDatasetPush(itemCount) ?? false;
+}
 exports.APIFY_DEFAULT_DATASET_ITEM_EVENT = 'apify-default-dataset-item';
 /** Scrapely-native synthetic event for default-dataset items. */
 exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = 'scrapely-default-dataset-item';
@@ -95,9 +137,77 @@ class ChargingManager {
     apifyClient;
     /** Set during actor shutdown; rejects new charge operations after the final flush. */
     _closed = false;
+    /** Sticky: the platform refused or capped a charge (HTTP 402 / fewer accepted) or the local gate trimmed one. */
+    chargeLimitHit = false;
+    /** Default-dataset rows refused after the limit (one warning, then a count). */
+    refusedDefaultDatasetItems = 0;
     /** @internal Called by Actor shutdown before the final flush. */
     close() {
         this._closed = true;
+        if (this.refusedDefaultDatasetItems > 0) {
+            console.warn(`[Scrapely] ${this.refusedDefaultDatasetItems} result(s) pushed to the default dataset after the spending limit was reached were not saved`);
+        }
+    }
+    /**
+     * Whether this pay-per-event run has used up its spending limit.
+     *
+     * True once a charge was refused or capped by the limit (the platform's
+     * HTTP 402 or a partial acceptance, or the local budget gate), or once the
+     * remaining budget cannot pay for one event of the cheapest priced type.
+     * From then on, pushes to the default dataset that are not paid for are
+     * refused. Always false for actors without pay-per-event pricing, without
+     * a limit, or with only free events.
+     */
+    isChargeLimitReached() {
+        if (this.chargingState === undefined || !this.isPayPerEvent)
+            return false;
+        if (this.chargeLimitHit)
+            return true;
+        if (!Number.isFinite(this.maxTotalChargeUsd))
+            return false;
+        const prices = Object.values(this.pricingInfo)
+            .map(({ price }) => price)
+            .filter((price) => price > 0);
+        if (prices.length === 0)
+            return false;
+        // Same rounding as the budget gate, so 0.03 - 0.02 still pays a $0.01 event.
+        return this.calculateMaxChargesByPrice(Math.min(...prices)) <= 0;
+    }
+    markChargeLimitHit() {
+        if (this.isPayPerEvent)
+            this.chargeLimitHit = true;
+    }
+    /** @internal Count rows dropped because of the limit; warn on the first one only. */
+    _noteRefusedDefaultDatasetItems(count) {
+        if (count <= 0)
+            return;
+        if (this.refusedDefaultDatasetItems === 0) {
+            const limit = Number.isFinite(this.maxTotalChargeUsd) ? `$${this.maxTotalChargeUsd}` : 'the limit';
+            console.warn(`[Scrapely] The spending limit of this run (maxTotalChargeUsd = ${limit}) has been reached: ` +
+                'results pushed to the default dataset are no longer saved. The Actor should stop now ' +
+                '(check ChargeResult.eventChargeLimitReached or Actor.getChargingManager().isChargeLimitReached()).');
+        }
+        this.refusedDefaultDatasetItems += count;
+    }
+    /** @internal */
+    _markChargeLimitHit() {
+        this.markChargeLimitHit();
+    }
+    /** @internal Whether every row of a push with these events is charged a positive price. */
+    _isPaidPerItem(eventName, isDefaultDataset) {
+        const names = [];
+        if (eventName !== undefined)
+            names.push(eventName);
+        if (isDefaultDataset)
+            names.push(resolveDefaultDatasetItemEventName(this.pricingInfo));
+        return names.some((name) => (this.calculateEventPrice(name) ?? 0) > 0);
+    }
+    /** @internal */
+    _refuseDefaultDatasetPush(itemCount) {
+        if (itemCount <= 0 || !this.isChargeLimitReached())
+            return false;
+        this._noteRefusedDefaultDatasetItems(itemCount);
+        return true;
     }
     /** @internal Whether shutdown has closed this manager to new charges. */
     isClosed() {
@@ -538,6 +648,8 @@ class ChargingManager {
                     try {
                         const accepted = await this.postChargeWithRetries(intent);
                         if (accepted !== null) {
+                            if (accepted < intent.count)
+                                this.markChargeLimitHit();
                             const eventPrice = this.getEventPricing(intent.eventName).price;
                             this.applyAcceptedCount(intent.eventName, intent.count, accepted, eventPrice);
                             this.pendingCharges.delete(intent.idempotencyKey);
@@ -546,6 +658,8 @@ class ChargingManager {
                     }
                     catch (err) {
                         if (err instanceof ChargeRejectedError) {
+                            if (isChargeLimitRejection(err))
+                                this.markChargeLimitHit();
                             const eventPrice = this.getEventPricing(intent.eventName).price;
                             this.applyAcceptedCount(intent.eventName, intent.count, 0, eventPrice);
                             this.pendingCharges.delete(intent.idempotencyKey);
@@ -613,6 +727,8 @@ class ChargingManager {
             }
             return 0;
         })();
+        if (chargedCount < count)
+            this.markChargeLimitHit();
         if (chargedCount === 0) {
             return {
                 eventChargeLimitReached: count > 0,
@@ -655,6 +771,8 @@ class ChargingManager {
             }
             catch (err) {
                 if (err instanceof ChargeRejectedError) {
+                    if (isChargeLimitRejection(err))
+                        this.markChargeLimitHit();
                     // Server definitively rejected: unwind local reservation.
                     this.pendingCharges.delete(intent.idempotencyKey);
                     chargedCount = this.applyAcceptedCount(normalizedEventName, chargedCount, 0, eventPricing.price);
@@ -670,6 +788,9 @@ class ChargingManager {
                 this.pendingCharges.delete(intent.idempotencyKey);
                 if (accepted > 0)
                     this.lastAcceptedOperationId = intent.idempotencyKey;
+                // The platform capped the charge at the run's limit.
+                if (accepted < chargedCount)
+                    this.markChargeLimitHit();
                 chargedCount = this.applyAcceptedCount(normalizedEventName, chargedCount, accepted, eventPricing.price);
             }
             else {
@@ -772,6 +893,22 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
         // A void that did not go through for an earlier failed push must land
         // before this push charges again, or one unsaved result is billed twice.
         const voidsConfirmed = (await chargingManager._retryPendingVoids?.()) !== false;
+        const requested = Array.isArray(items) ? items.length : 1;
+        // Charge-limit safety net: once the run's spending limit is used up, a
+        // default-dataset push that does not pay per row (no event, or only free
+        // events) stores nothing. Paid pushes go on below, where the budget cap
+        // stores only the rows that are actually charged.
+        if (isDefaultDataset &&
+            requested > 0 &&
+            chargingManager.isChargeLimitReached?.() &&
+            !chargingManager._isPaidPerItem(eventName, isDefaultDataset)) {
+            chargingManager._noteRefusedDefaultDatasetItems(requested);
+            return {
+                eventChargeLimitReached: true,
+                chargedCount: 0,
+                chargeableWithinLimit: chargingManager._chargeableWithinLimit?.() ?? {},
+            };
+        }
         const { limitedItems, eventsToCharge } = chargingManager.calculatePushDataLimits({
             items,
             eventName,
@@ -787,6 +924,11 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
         }
         const itemsArray = Array.isArray(items) ? items : [items];
         const allItemsTrimmed = itemsArray.length > 0 && limitedItems.length === 0;
+        if (isDefaultDataset && limitedItems.length < itemsArray.length) {
+            // Rows beyond what the budget can pay for are dropped.
+            chargingManager._markChargeLimitHit?.();
+            chargingManager._noteRefusedDefaultDatasetItems?.(itemsArray.length - limitedItems.length);
+        }
         if (limitedItems.length === 0) {
             return {
                 eventChargeLimitReached: allItemsTrimmed,
@@ -872,7 +1014,12 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                     };
                 }
             }
-            chargeResult = results.reduce(mergeChargeResults);
+            // Report rows charged, as Apify does for the pushed event: summing the
+            // explicit and default-dataset events would count every row twice.
+            chargeResult = { ...results.reduce(mergeChargeResults), chargedCount: acceptedForItems };
+            if (isDefaultDataset && acceptedForItems < limitedItems.length) {
+                chargingManager._noteRefusedDefaultDatasetItems?.(limitedItems.length - acceptedForItems);
+            }
             if (acceptedForItems <= 0) {
                 // Nothing will be stored (another event of this push was capped to
                 // zero): undo every platform charge this operation made, so no
@@ -896,7 +1043,7 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
             }
             const toPersist = limitedItems.slice(0, acceptedForItems);
             try {
-                await pushFn(Array.isArray(items) ? toPersist : toPersist[0]);
+                await chargeGatedPush.run(true, () => pushFn(Array.isArray(items) ? toPersist : toPersist[0]));
             }
             catch (pushErr) {
                 // Storage failed after the charge was accepted: a result is billed
@@ -932,7 +1079,7 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
             return chargeResult;
         }
         // No billable events — still persist the budget-limited items.
-        await pushFn(Array.isArray(items) ? limitedItems : limitedItems[0]);
+        await chargeGatedPush.run(true, () => pushFn(Array.isArray(items) ? limitedItems : limitedItems[0]));
         return {
             eventChargeLimitReached: false,
             chargedCount: 0,

package/dist/charging_client.js

@@ -17,7 +17,11 @@ function createChargingHooks(actor) {
         // Called by the client at `dataset()` time, so pricing is consulted
         // lazily — same timing as the former subclass override.
         shouldChargeDatasetPushes() {
-            return (0, charging_js_1.hasDefaultDatasetItemEventInPricing)(actor.getChargingManager().getPricingInfo().perEventPrices);
+            const chargingManager = actor.getChargingManager();
+            // Past the spending limit, route default-dataset pushes through
+            // pushDataAndCharge too, which refuses rows nobody pays for.
+            return ((0, charging_js_1.hasDefaultDatasetItemEventInPricing)(chargingManager.getPricingInfo().perEventPrices) ||
+                (chargingManager.isChargeLimitReached?.() ?? false));
         },
         async pushAndCharge({ items, eventName, pushFn }) {
             return (0, charging_js_1.pushDataAndCharge)({

package/dist/storage/api-storage-client.js

@@ -5,6 +5,7 @@ exports.isRetryableResponse = isRetryableResponse;
 exports.serverRetryDelayMs = serverRetryDelayMs;
 exports.isConnectFailure = isConnectFailure;
 const node_crypto_1 = require("node:crypto");
+const charging_js_1 = require("../charging.js");
 const kvs_iteration_js_1 = require("./kvs-iteration.js");
 function buildRqQueryParams(baseClientKey, options) {
     const params = new URLSearchParams();
@@ -559,6 +560,10 @@ class ApiDatasetClient extends BaseApiClient {
         else {
             parsed = [items];
         }
+        // Charge-limit safety net: a pay-per-event run past its spending limit
+        // must not store unbilled rows in its default dataset.
+        if (this.id === (0, charging_js_1.defaultDatasetIdFromEnv)() && (0, charging_js_1.refuseDefaultDatasetPush)(parsed.length))
+            return;
         await this.request('POST', `/datasets/${this.id}/items`, parsed);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.14",
+    "version": "1.2.16",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -22,6 +22,7 @@
     "scripts": {
         "test": "npx tsx tests/test-mvp.ts",
         "test:charging": "npx tsx --test tests/charging-pending.test.ts",
+        "test:charge-limit-guard": "npx tsx --test tests/charge-limit-guard.test.ts",
         "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
         "test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
         "test:charging-client": "npx tsx --test tests/charging-client.test.ts",