scrapely 1.2.23 → 1.2.25

package/dist/actor.d.ts

@@ -21,8 +21,14 @@ import { type ApiStorageConfig } from './storage/api-storage-client.js';
 import { ChargingManager, type ChargeOptions as ChargingChargeOptions, type ChargeResult as ChargingChargeResult } from './charging.js';
 /** Options for a multi-event charged push (`Actor.pushData(row, { charges })`). */
 export interface PushDataChargeOptions {
-    /** Events per row, e.g. `{ result: 1, page: 3 }`. */
+    /** Events per row, e.g. `{ result: 1, page: 3 }` (per batch with `chargePer: 'batch'`). */
     charges: Record<string, number>;
+    /**
+     * 'row' (default): `charges` for every row. 'batch': once for the whole push, e.g.
+     * `pushData(pageRows, { charges: { document: 1 }, chargePer: 'batch' })`: all rows are
+     * stored if the charge is accepted, none if it is refused.
+     */
+    chargePer?: 'row' | 'batch';
 }
 import { createProxyConfiguration, type CreateProxyConfigurationOptions } from './proxy-configuration.js';
 import { type BrowserConfiguration, type BrowserConfigurationOptions } from './browser-configuration.js';

package/dist/actor.js

@@ -526,6 +526,7 @@ class Actor {
                 chargingManager: this.chargingManager,
                 items: item,
                 charges: eventNameOrOptions.charges,
+                chargePer: eventNameOrOptions.chargePer ?? 'row',
                 isDefaultDataset: dataset.id === this.config.get('defaultDatasetId'),
                 pushFn: async (rows) => { await dataset.pushData(rows); },
             });

package/dist/charging.d.ts

@@ -93,10 +93,22 @@ export declare class ChargingManager {
      */
     private platformOtherUsd;
     private lastRemainingBudgetNd;
+    /** True once the platform reported a remaining budget (its usage is then in the local budget). */
+    private platformBudgetSeen;
     /** maxTotalChargeUsd minus this SDK's charges and the platform's other run spend. */
     private remainingBudgetUsd;
     /** Fold the last charge response's remainingBudgetNd into the local budget (monotonic). */
     private syncPlatformBudget;
+    /**
+     * @internal Whole rows a multi-event retry can pay for after a short answer:
+     * what the voided attempt was accepted (budget the void gives back) at the
+     * full per-row cost, or the local remaining budget once the platform has
+     * reported it (then it includes the leftover after the short answer).
+     */
+    _rowsAfterShort(attempt: Array<{
+        name: string;
+        charged: number;
+    }>, perRow: Record<string, number>): number;
     /** @internal Platform spend outside this SDK's charges (tests). */
     _platformOtherUsd(): number;
     /** @internal Called by Actor shutdown before the final flush. */
@@ -128,7 +140,7 @@ export declare class ChargingManager {
      * default-dataset-item event on the default dataset when it is priced, and
      * how many rows the remaining budget pays for in full.
      */
-    _rowCharges(charges: Record<string, number>, isDefaultDataset: boolean, rowCount: number): {
+    _rowCharges(charges: Record<string, number>, isDefaultDataset: boolean, rowCount: number, batch?: boolean): {
         perRow: Record<string, number>;
         rows: number;
     };
@@ -284,12 +296,14 @@ export declare function validateRowCharges(charges: unknown): Record<string, num
  * the rows that are kept, so no row is stored part-paid and no event is billed
  * for a row that was not stored. Unpriced events are ignored.
  */
-export declare function pushDataWithCharges<T>({ chargingManager, items, charges, isDefaultDataset, pushFn, }: {
+export declare function pushDataWithCharges<T>({ chargingManager, items, charges, isDefaultDataset, pushFn, chargePer, }: {
     chargingManager: ChargingManager;
     items: T | T[];
     charges: Record<string, number>;
     isDefaultDataset: boolean;
     pushFn: (rows: T | T[]) => Promise<void>;
+    /** 'batch': charge `charges` once for the whole push; all rows stored if accepted, none if refused. */
+    chargePer?: 'row' | 'batch';
 }): Promise<ChargeResult>;
 /**
  * @internal True when a dataset push certainly stored nothing: the storage API

package/dist/charging.js

@@ -172,6 +172,8 @@ class ChargingManager {
      */
     platformOtherUsd = 0;
     lastRemainingBudgetNd = null;
+    /** True once the platform reported a remaining budget (its usage is then in the local budget). */
+    platformBudgetSeen = false;
     /** maxTotalChargeUsd minus this SDK's charges and the platform's other run spend. */
     remainingBudgetUsd() {
         return this.maxTotalChargeUsd - this.calculateTotalChargedAmount() - this.platformOtherUsd;
@@ -182,10 +184,28 @@ class ChargingManager {
         this.lastRemainingBudgetNd = null;
         if (nd === null || nd < 0 || !Number.isFinite(this.maxTotalChargeUsd))
             return;
+        this.platformBudgetSeen = true;
         const other = Number((this.maxTotalChargeUsd - nd / 1e9 - this.calculateTotalChargedAmount()).toFixed(9));
         if (other > this.platformOtherUsd)
             this.platformOtherUsd = other;
     }
+    /**
+     * @internal Whole rows a multi-event retry can pay for after a short answer:
+     * what the voided attempt was accepted (budget the void gives back) at the
+     * full per-row cost, or the local remaining budget once the platform has
+     * reported it (then it includes the leftover after the short answer).
+     */
+    _rowsAfterShort(attempt, perRow) {
+        const rowCost = Object.entries(perRow).reduce((a, [n, c]) => a + (this.calculateEventPrice(n) ?? 0) * c, 0);
+        if (rowCost <= 0)
+            return 0;
+        const spent = attempt.reduce((a, op) => a + (this.calculateEventPrice(op.name) ?? 0) * op.charged, 0);
+        let fit = Math.floor(Number((spent / rowCost).toFixed(4)));
+        if (this.platformBudgetSeen && Number.isFinite(this.maxTotalChargeUsd)) {
+            fit = Math.max(fit, Math.floor(Number((this.remainingBudgetUsd() / rowCost).toFixed(4))));
+        }
+        return Math.max(0, fit);
+    }
     /** @internal Platform spend outside this SDK's charges (tests). */
     _platformOtherUsd() {
         return this.platformOtherUsd;
@@ -266,7 +286,7 @@ class ChargingManager {
      * default-dataset-item event on the default dataset when it is priced, and
      * how many rows the remaining budget pays for in full.
      */
-    _rowCharges(charges, isDefaultDataset, rowCount) {
+    _rowCharges(charges, isDefaultDataset, rowCount, batch = false) {
         const perRow = {};
         for (const [name, n] of Object.entries(charges)) {
             if ((this.calculateEventPrice(name) ?? 0) > 0)
@@ -274,11 +294,13 @@ class ChargingManager {
         }
         if (isDefaultDataset) {
             const def = resolveDefaultDatasetItemEventName(this.pricingInfo);
+            // Per stored row, also for a batch.
             if (!(def in perRow) && (this.calculateEventPrice(def) ?? 0) > 0)
-                perRow[def] = 1;
+                perRow[def] = batch ? rowCount : 1;
         }
         const rowCost = Object.entries(perRow).reduce((a, [name, n]) => a + (this.calculateEventPrice(name) ?? 0) * n, 0);
-        let rows = rowCount;
+        // Units charged for: rows, or the one batch (rowCost is then the batch's cost).
+        let rows = batch ? 1 : rowCount;
         if (rowCost > 0 && Number.isFinite(this.maxTotalChargeUsd)) {
             const fit = Math.floor(Number((this.remainingBudgetUsd() / rowCost).toFixed(4)));
             rows = Math.min(rows, Math.max(0, fit));
@@ -1317,8 +1339,11 @@ function validateRowCharges(charges) {
  * the rows that are kept, so no row is stored part-paid and no event is billed
  * for a row that was not stored. Unpriced events are ignored.
  */
-async function pushDataWithCharges({ chargingManager, items, charges, isDefaultDataset, pushFn, }) {
+async function pushDataWithCharges({ chargingManager, items, charges, isDefaultDataset, pushFn, chargePer = 'row', }) {
     const perRowIn = validateRowCharges(charges);
+    if (chargePer !== 'row' && chargePer !== 'batch')
+        throw new Error("chargePer must be 'row' or 'batch'");
+    const batch = chargePer === 'batch';
     const rowsIn = Array.isArray(items) ? items : [items];
     const single = !Array.isArray(items);
     const store = (rows) => chargeGatedPush.run(true, () => pushFn(single ? rows[0] : rows));
@@ -1333,7 +1358,7 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
             await store(rowsIn);
             return { eventChargeLimitReached: false, chargedCount: 0, chargeableWithinLimit: {} };
         }
-        let { perRow, rows } = chargingManager._rowCharges(perRowIn, isDefaultDataset, rowsIn.length);
+        let { perRow, rows } = chargingManager._rowCharges(perRowIn, isDefaultDataset, rowsIn.length, batch);
         if (Object.keys(perRow).length === 0) {
             // Only free events: an unpaid push (prepaid-row rules apply).
             const pastLimit = chargingManager.isChargeLimitReached();
@@ -1361,7 +1386,7 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
         };
         const order = Object.entries(perRow).sort(([a], [b]) => Number(isSyntheticEvent(a)) - Number(isSyntheticEvent(b)) || a.localeCompare(b));
         let ops = [];
-        for (let attempt = 0; attempt < 4 && rows > 0; attempt++) {
+        for (let attempt = 0; attempt < 6 && rows > 0; attempt++) {
             ops = [];
             let shortTo = null;
             for (const [name, n] of order) {
@@ -1389,10 +1414,14 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
             }
             if (shortTo === null)
                 break;
-            // An event came back short: void this attempt and retry with the rows it pays for.
+            // An event came back short: void this attempt and retry with the whole
+            // rows the budget covers at the FULL per-row cost (sizing from the short
+            // event alone came back short again: appstore 64df0a3608d74af88, 0 saved).
+            // At least one single-row try: the last affordable row may still fit.
+            const attemptOps = ops.map((op) => ({ name: op.name, charged: op.charged }));
             await undo(ops);
             ops = [];
-            rows = shortTo;
+            rows = Math.min(rows - 1, Math.max(chargingManager._rowsAfterShort(attemptOps, perRow), 1));
         }
         if (rows <= 0 || ops.length === 0) {
             await undo(ops);
@@ -1401,23 +1430,24 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
                 chargingManager._noteRefusedDefaultDatasetItems(rowsIn.length);
             return { eventChargeLimitReached: true, chargedCount: 0, chargeableWithinLimit: chargingManager._chargeableWithinLimit() };
         }
-        if (rows < rowsIn.length) {
+        const kept = batch ? rowsIn : rowsIn.slice(0, rows);
+        if (kept.length < rowsIn.length) {
             chargingManager._markChargeLimitHit();
             if (isDefaultDataset)
-                chargingManager._noteRefusedDefaultDatasetItems(rowsIn.length - rows);
+                chargingManager._noteRefusedDefaultDatasetItems(rowsIn.length - kept.length);
         }
         try {
-            await store(rowsIn.slice(0, rows));
+            await store(kept);
         }
         catch (pushErr) {
-            console.error(`[Scrapely] Dataset push failed after charges accepted (rows=${rows}): ${pushErr.message}; voiding the charges`);
+            console.error(`[Scrapely] Dataset push failed after charges accepted (rows=${kept.length}): ${pushErr.message}; voiding the charges`);
             if (pushCertainlyFailed(pushErr))
                 await undo(ops);
             throw pushErr;
         }
         return {
-            eventChargeLimitReached: rows < rowsIn.length || chargingManager.isChargeLimitReached(),
-            chargedCount: rows,
+            eventChargeLimitReached: kept.length < rowsIn.length || chargingManager.isChargeLimitReached(),
+            chargedCount: kept.length,
             chargeableWithinLimit: chargingManager._chargeableWithinLimit(),
         };
     });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.23",
+    "version": "1.2.25",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",