scrapely 1.2.24 → 1.2.26

package/dist/actor.d.ts

@@ -21,8 +21,24 @@ import { type ApiStorageConfig } from './storage/api-storage-client.js';
 import { ChargingManager, type ChargeOptions as ChargingChargeOptions, type ChargeResult as ChargingChargeResult } from './charging.js';
 /** Options for a multi-event charged push (`Actor.pushData(row, { charges })`). */
 export interface PushDataChargeOptions {
-    /** Events per row, e.g. `{ result: 1, page: 3 }`. */
-    charges: Record<string, number>;
+    /** Events per row, e.g. `{ result: 1, page: 3 }` (per batch with `chargePer: 'batch'`). */
+    charges?: Record<string, number>;
+    /** One event per row (same as `pushData(item, eventName)`); not together with `charges`. */
+    eventName?: string;
+    /**
+     * Push to this named dataset (name, or a Dataset from `Actor.openDataset`) instead of the
+     * default one, with the same charge-first semantics, e.g.
+     * `pushData(rows, { eventName: 'comment', dataset: \`comments-${runId}\` })`. Rows of a named
+     * dataset carry no implicit default-dataset-item event and are not gated by the default
+     * dataset's spending-limit safety net; charged pushes store only the rows that were charged.
+     */
+    dataset?: string | Dataset;
+    /**
+     * 'row' (default): `charges` for every row. 'batch': once for the whole push, e.g.
+     * `pushData(pageRows, { charges: { document: 1 }, chargePer: 'batch' })`: all rows are
+     * stored if the charge is accepted, none if it is refused.
+     */
+    chargePer?: 'row' | 'batch';
 }
 import { createProxyConfiguration, type CreateProxyConfigurationOptions } from './proxy-configuration.js';
 import { type BrowserConfiguration, type BrowserConfigurationOptions } from './browser-configuration.js';

package/dist/actor.js

@@ -520,13 +520,41 @@ class Actor {
     async pushData(item, eventNameOrOptions) {
         Actor._ensureActorInit('pushData');
         if (eventNameOrOptions !== undefined && typeof eventNameOrOptions === 'object') {
+            const opts = eventNameOrOptions;
             const { Dataset } = await import('crawlee');
-            const dataset = await Dataset.open();
-            return (0, charging_js_2.pushDataWithCharges)({
+            const dataset = opts.dataset === undefined
+                ? await Dataset.open()
+                : typeof opts.dataset === 'string' ? await Dataset.open(opts.dataset) : opts.dataset;
+            const isDefaultDataset = dataset.id === this.config.get('defaultDatasetId');
+            if (opts.charges !== undefined) {
+                if (opts.eventName !== undefined)
+                    throw new Error('Pass either eventName or charges, not both');
+                return (0, charging_js_2.pushDataWithCharges)({
+                    chargingManager: this.chargingManager,
+                    items: item,
+                    charges: opts.charges,
+                    chargePer: opts.chargePer ?? 'row',
+                    isDefaultDataset,
+                    pushFn: async (rows) => { await dataset.pushData(rows); },
+                });
+            }
+            if (opts.chargePer !== undefined && opts.chargePer !== 'row')
+                throw new Error("chargePer needs charges: {...}");
+            if (isDefaultDataset)
+                return this.pushData(item, opts.eventName);
+            if (opts.eventName === 'apify-default-dataset-item' || opts.eventName === 'scrapely-default-dataset-item') {
+                throw new Error(`Cannot charge for synthetic event '${opts.eventName}' manually`);
+            }
+            if (opts.eventName === undefined) {
+                // A plain push to a named dataset: not gated by the default dataset's safety net.
+                await dataset.pushData(item);
+                return;
+            }
+            return (0, charging_js_1.pushDataAndCharge)({
                 chargingManager: this.chargingManager,
                 items: item,
-                charges: eventNameOrOptions.charges,
-                isDefaultDataset: dataset.id === this.config.get('defaultDatasetId'),
+                eventName: opts.eventName,
+                isDefaultDataset: false,
                 pushFn: async (rows) => { await dataset.pushData(rows); },
             });
         }

package/dist/charging.d.ts

@@ -140,7 +140,7 @@ export declare class ChargingManager {
      * default-dataset-item event on the default dataset when it is priced, and
      * how many rows the remaining budget pays for in full.
      */
-    _rowCharges(charges: Record<string, number>, isDefaultDataset: boolean, rowCount: number): {
+    _rowCharges(charges: Record<string, number>, isDefaultDataset: boolean, rowCount: number, batch?: boolean): {
         perRow: Record<string, number>;
         rows: number;
     };
@@ -296,12 +296,14 @@ export declare function validateRowCharges(charges: unknown): Record<string, num
  * the rows that are kept, so no row is stored part-paid and no event is billed
  * for a row that was not stored. Unpriced events are ignored.
  */
-export declare function pushDataWithCharges<T>({ chargingManager, items, charges, isDefaultDataset, pushFn, }: {
+export declare function pushDataWithCharges<T>({ chargingManager, items, charges, isDefaultDataset, pushFn, chargePer, }: {
     chargingManager: ChargingManager;
     items: T | T[];
     charges: Record<string, number>;
     isDefaultDataset: boolean;
     pushFn: (rows: T | T[]) => Promise<void>;
+    /** 'batch': charge `charges` once for the whole push; all rows stored if accepted, none if refused. */
+    chargePer?: 'row' | 'batch';
 }): Promise<ChargeResult>;
 /**
  * @internal True when a dataset push certainly stored nothing: the storage API

package/dist/charging.js

@@ -286,7 +286,7 @@ class ChargingManager {
      * default-dataset-item event on the default dataset when it is priced, and
      * how many rows the remaining budget pays for in full.
      */
-    _rowCharges(charges, isDefaultDataset, rowCount) {
+    _rowCharges(charges, isDefaultDataset, rowCount, batch = false) {
         const perRow = {};
         for (const [name, n] of Object.entries(charges)) {
             if ((this.calculateEventPrice(name) ?? 0) > 0)
@@ -294,11 +294,13 @@ class ChargingManager {
         }
         if (isDefaultDataset) {
             const def = resolveDefaultDatasetItemEventName(this.pricingInfo);
+            // Per stored row, also for a batch.
             if (!(def in perRow) && (this.calculateEventPrice(def) ?? 0) > 0)
-                perRow[def] = 1;
+                perRow[def] = batch ? rowCount : 1;
         }
         const rowCost = Object.entries(perRow).reduce((a, [name, n]) => a + (this.calculateEventPrice(name) ?? 0) * n, 0);
-        let rows = rowCount;
+        // Units charged for: rows, or the one batch (rowCost is then the batch's cost).
+        let rows = batch ? 1 : rowCount;
         if (rowCost > 0 && Number.isFinite(this.maxTotalChargeUsd)) {
             const fit = Math.floor(Number((this.remainingBudgetUsd() / rowCost).toFixed(4)));
             rows = Math.min(rows, Math.max(0, fit));
@@ -1337,8 +1339,11 @@ function validateRowCharges(charges) {
  * the rows that are kept, so no row is stored part-paid and no event is billed
  * for a row that was not stored. Unpriced events are ignored.
  */
-async function pushDataWithCharges({ chargingManager, items, charges, isDefaultDataset, pushFn, }) {
+async function pushDataWithCharges({ chargingManager, items, charges, isDefaultDataset, pushFn, chargePer = 'row', }) {
     const perRowIn = validateRowCharges(charges);
+    if (chargePer !== 'row' && chargePer !== 'batch')
+        throw new Error("chargePer must be 'row' or 'batch'");
+    const batch = chargePer === 'batch';
     const rowsIn = Array.isArray(items) ? items : [items];
     const single = !Array.isArray(items);
     const store = (rows) => chargeGatedPush.run(true, () => pushFn(single ? rows[0] : rows));
@@ -1353,7 +1358,7 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
             await store(rowsIn);
             return { eventChargeLimitReached: false, chargedCount: 0, chargeableWithinLimit: {} };
         }
-        let { perRow, rows } = chargingManager._rowCharges(perRowIn, isDefaultDataset, rowsIn.length);
+        let { perRow, rows } = chargingManager._rowCharges(perRowIn, isDefaultDataset, rowsIn.length, batch);
         if (Object.keys(perRow).length === 0) {
             // Only free events: an unpaid push (prepaid-row rules apply).
             const pastLimit = chargingManager.isChargeLimitReached();
@@ -1425,23 +1430,24 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
                 chargingManager._noteRefusedDefaultDatasetItems(rowsIn.length);
             return { eventChargeLimitReached: true, chargedCount: 0, chargeableWithinLimit: chargingManager._chargeableWithinLimit() };
         }
-        if (rows < rowsIn.length) {
+        const kept = batch ? rowsIn : rowsIn.slice(0, rows);
+        if (kept.length < rowsIn.length) {
             chargingManager._markChargeLimitHit();
             if (isDefaultDataset)
-                chargingManager._noteRefusedDefaultDatasetItems(rowsIn.length - rows);
+                chargingManager._noteRefusedDefaultDatasetItems(rowsIn.length - kept.length);
         }
         try {
-            await store(rowsIn.slice(0, rows));
+            await store(kept);
         }
         catch (pushErr) {
-            console.error(`[Scrapely] Dataset push failed after charges accepted (rows=${rows}): ${pushErr.message}; voiding the charges`);
+            console.error(`[Scrapely] Dataset push failed after charges accepted (rows=${kept.length}): ${pushErr.message}; voiding the charges`);
             if (pushCertainlyFailed(pushErr))
                 await undo(ops);
             throw pushErr;
         }
         return {
-            eventChargeLimitReached: rows < rowsIn.length || chargingManager.isChargeLimitReached(),
-            chargedCount: rows,
+            eventChargeLimitReached: kept.length < rowsIn.length || chargingManager.isChargeLimitReached(),
+            chargedCount: kept.length,
             chargeableWithinLimit: chargingManager._chargeableWithinLimit(),
         };
     });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.24",
+    "version": "1.2.26",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",