scrapely 1.2.25 → 1.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +11 -1
- package/dist/actor.js +32 -5
- package/package.json +1 -1
package/dist/actor.d.ts
CHANGED
|
@@ -22,7 +22,17 @@ import { ChargingManager, type ChargeOptions as ChargingChargeOptions, type Char
|
|
|
22
22
|
/** Options for a multi-event charged push (`Actor.pushData(row, { charges })`). */
|
|
23
23
|
export interface PushDataChargeOptions {
|
|
24
24
|
/** Events per row, e.g. `{ result: 1, page: 3 }` (per batch with `chargePer: 'batch'`). */
|
|
25
|
-
charges
|
|
25
|
+
charges?: Record<string, number>;
|
|
26
|
+
/** One event per row (same as `pushData(item, eventName)`); not together with `charges`. */
|
|
27
|
+
eventName?: string;
|
|
28
|
+
/**
|
|
29
|
+
* Push to this named dataset (name, or a Dataset from `Actor.openDataset`) instead of the
|
|
30
|
+
* default one, with the same charge-first semantics, e.g.
|
|
31
|
+
* `pushData(rows, { eventName: 'comment', dataset: \`comments-${runId}\` })`. Rows of a named
|
|
32
|
+
* dataset carry no implicit default-dataset-item event and are not gated by the default
|
|
33
|
+
* dataset's spending-limit safety net; charged pushes store only the rows that were charged.
|
|
34
|
+
*/
|
|
35
|
+
dataset?: string | Dataset;
|
|
26
36
|
/**
|
|
27
37
|
* 'row' (default): `charges` for every row. 'batch': once for the whole push, e.g.
|
|
28
38
|
* `pushData(pageRows, { charges: { document: 1 }, chargePer: 'batch' })`: all rows are
|
package/dist/actor.js
CHANGED
|
@@ -520,14 +520,41 @@ class Actor {
|
|
|
520
520
|
async pushData(item, eventNameOrOptions) {
|
|
521
521
|
Actor._ensureActorInit('pushData');
|
|
522
522
|
if (eventNameOrOptions !== undefined && typeof eventNameOrOptions === 'object') {
|
|
523
|
+
const opts = eventNameOrOptions;
|
|
523
524
|
const { Dataset } = await import('crawlee');
|
|
524
|
-
const dataset =
|
|
525
|
-
|
|
525
|
+
const dataset = opts.dataset === undefined
|
|
526
|
+
? await Dataset.open()
|
|
527
|
+
: typeof opts.dataset === 'string' ? await Dataset.open(opts.dataset) : opts.dataset;
|
|
528
|
+
const isDefaultDataset = dataset.id === this.config.get('defaultDatasetId');
|
|
529
|
+
if (opts.charges !== undefined) {
|
|
530
|
+
if (opts.eventName !== undefined)
|
|
531
|
+
throw new Error('Pass either eventName or charges, not both');
|
|
532
|
+
return (0, charging_js_2.pushDataWithCharges)({
|
|
533
|
+
chargingManager: this.chargingManager,
|
|
534
|
+
items: item,
|
|
535
|
+
charges: opts.charges,
|
|
536
|
+
chargePer: opts.chargePer ?? 'row',
|
|
537
|
+
isDefaultDataset,
|
|
538
|
+
pushFn: async (rows) => { await dataset.pushData(rows); },
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
if (opts.chargePer !== undefined && opts.chargePer !== 'row')
|
|
542
|
+
throw new Error("chargePer needs charges: {...}");
|
|
543
|
+
if (isDefaultDataset)
|
|
544
|
+
return this.pushData(item, opts.eventName);
|
|
545
|
+
if (opts.eventName === 'apify-default-dataset-item' || opts.eventName === 'scrapely-default-dataset-item') {
|
|
546
|
+
throw new Error(`Cannot charge for synthetic event '${opts.eventName}' manually`);
|
|
547
|
+
}
|
|
548
|
+
if (opts.eventName === undefined) {
|
|
549
|
+
// A plain push to a named dataset: not gated by the default dataset's safety net.
|
|
550
|
+
await dataset.pushData(item);
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
return (0, charging_js_1.pushDataAndCharge)({
|
|
526
554
|
chargingManager: this.chargingManager,
|
|
527
555
|
items: item,
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
isDefaultDataset: dataset.id === this.config.get('defaultDatasetId'),
|
|
556
|
+
eventName: opts.eventName,
|
|
557
|
+
isDefaultDataset: false,
|
|
531
558
|
pushFn: async (rows) => { await dataset.pushData(rows); },
|
|
532
559
|
});
|
|
533
560
|
}
|