scrapely 1.2.8 → 1.2.10

package/dist/actor.d.ts

@@ -371,7 +371,9 @@ export declare class Actor {
     static getInputOrThrow<T = ActorInput>(): Promise<T>;
     /**
      * Synchronously get the actor input (cached from first access).
-     * This matches Apify's Actor.input behavior.
+     * The platform sets ACTOR_INPUT only for small inputs; for large ones this
+     * returns the input once `await Actor.getInput()` has loaded it from the
+     * default key-value store.
      */
     static get input(): Record<string, any> | undefined;
     /**
@@ -701,3 +703,5 @@ export declare class Actor {
         enqueueLinks: typeof import("./utils.js").enqueueLinks;
     };
 }
+/** Resolves once stdout and stderr have flushed their buffers, or after `maxMs`. */
+export declare function waitForStdioDrain(maxMs: number, streams?: NodeJS.WritableStream[]): Promise<void>;

package/dist/actor.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Actor = exports.EXIT_CODES = exports.ACTOR_PERMISSION_LEVEL = void 0;
+exports.waitForStdioDrain = waitForStdioDrain;
 require("./log-env.js");
 const node_events_1 = require("node:events");
 /** Actor permission level - matches Apify's ACTOR_PERMISSION_LEVEL enum */
@@ -172,6 +173,7 @@ class Actor {
     async init(options = {}) {
         if (this.initialized)
             return;
+        cachedInput = undefined; // a fresh init reads this run's input again
         // NOTE: `initialized` is set at the END of init — a failed platform init
         // must stay retryable instead of leaving a half-initialized Actor.
         Actor.installFatalChargeHandlers();
@@ -410,6 +412,13 @@ class Actor {
                 const metamorphKey = `INPUT-METAMORPH-${count}`;
                 const metamorphInput = await store.getValue(metamorphKey);
                 if (metamorphInput !== null && metamorphInput !== undefined) {
+                    // Same treatment as the regular input: decrypt secret fields
+                    // and let the synchronous Actor.input getter return it.
+                    if (typeof metamorphInput === 'object' && !Array.isArray(metamorphInput)) {
+                        const decrypted = (0, secrets_js_1.decryptSecretInputs)(metamorphInput);
+                        cachedInput = decrypted;
+                        return decrypted;
+                    }
                     return metamorphInput;
                 }
                 // Fall through to regular INPUT if metamorph key not found
@@ -437,7 +446,11 @@ class Actor {
         // Decrypt secret inputs after loading (must happen here, not in init(),
         // because the input needs to be available first)
         if (input && typeof input === 'object' && !Array.isArray(input)) {
-            return (0, secrets_js_1.decryptSecretInputs)(input);
+            const decrypted = (0, secrets_js_1.decryptSecretInputs)(input);
+            // Large inputs are only in the INPUT record (no ACTOR_INPUT env):
+            // let the synchronous Actor.input getter return what was loaded here.
+            cachedInput = decrypted;
+            return decrypted;
         }
         return input;
     }
@@ -450,14 +463,16 @@ class Actor {
     }
     /**
      * Synchronously get the actor input (cached from first access).
-     * This matches Apify's Actor.input behavior.
+     * The platform sets ACTOR_INPUT only for small inputs; for large ones this
+     * returns the input once `await Actor.getInput()` has loaded it from the
+     * default key-value store.
      */
     static get input() {
         if (cachedInput !== undefined)
             return cachedInput ?? undefined;
         const raw = process.env.ACTOR_INPUT;
         if (!raw) {
-            cachedInput = null;
+            // Not cached: a later Actor.getInput() fills it from the INPUT record.
             return undefined;
         }
         try {
@@ -928,10 +943,11 @@ class Actor {
         await eventManager.close();
         (0, events_client_js_1.disconnectEvents)();
         (0, patch_storage_js_1.disconnectStorage)();
-        // Brief grace for in-flight I/O after flush (bounded).
-        await new Promise((resolve) => {
-            setTimeout(resolve, Math.min(1000, flushDeadlineMs));
-        });
+        // Let buffered stdout/stderr reach the log pipe before the process exits
+        // (pipe writes are async in Node). Storage was flushed above, so this
+        // replaces the old fixed 1 s sleep: it resolves as soon as both streams
+        // drain, usually within milliseconds, and is still bounded.
+        await waitForStdioDrain(Math.min(1000, flushDeadlineMs));
     }
     /**
      * Sets the status message for the current Actor run.
@@ -1439,3 +1455,22 @@ class Actor {
     static utils = utils_js_1.utils;
 }
 exports.Actor = Actor;
+/** Resolves once stdout and stderr have flushed their buffers, or after `maxMs`. */
+async function waitForStdioDrain(maxMs, streams = [process.stdout, process.stderr]) {
+    const drained = Promise.all(streams.map((stream) => new Promise((resolve) => {
+        const s = stream;
+        if (!s.writableLength || s.writableFinished)
+            return resolve();
+        // An ended stream accepts no more writes; wait for it to finish flushing.
+        if (s.writableEnded) {
+            s.once('finish', () => resolve());
+            return;
+        }
+        // A zero-length write's callback runs after everything queued before it.
+        s.write('', () => resolve());
+    })));
+    let timer;
+    await Promise.race([drained, new Promise((resolve) => { timer = setTimeout(resolve, maxMs); })]);
+    if (timer)
+        clearTimeout(timer);
+}

package/dist/charging.d.ts

@@ -76,6 +76,10 @@ export declare class ChargingManager {
     private chargingState?;
     private chargingLogDataset?;
     private notPpeWarningPrinted;
+    /** Idempotency key of the last charge the platform accepted (pushDataAndCharge void-on-failure). */
+    private lastAcceptedOperationId;
+    /** Voids not yet confirmed by the platform; retried before the next push and at exit. */
+    private pendingVoids;
     /** Unresolved platform POSTs: key = idempotency UUID (never event name). */
     private readonly pendingCharges;
     private flushInFlight;
@@ -118,6 +122,8 @@ export declare class ChargingManager {
     private getEventPricing;
     private normalizeChargingEventName;
     private calculateMaxChargesByPrice;
+    /** @internal Remaining allowance per event (pushDataAndCharge's uncertain path). */
+    _chargeableWithinLimit(): Record<string, number>;
     private getChargeableWithinLimit;
     private quarantineStalePending;
     /**
@@ -126,6 +132,26 @@ export declare class ChargingManager {
      * Returns null when the outcome is uncertain after retries.
      */
     private postChargeWithRetries;
+    /** @internal Idempotency key of the charge accepted by the last _chargeExclusive call, if any. */
+    _takeLastAcceptedOperationId(): string | undefined;
+    /**
+     * @internal Reverse an accepted charge whose items were never stored
+     * (POST /v2/actor-runs/:runId/charge/void), then unwind the local
+     * reservation. Returns true when the platform voided it.
+     */
+    /**
+     * @internal Cancel a charge whose outcome was never confirmed, for items that
+     * were then not stored: drop the pending intent (so the exit flush never
+     * posts it) and void it in case the original request did apply. A 404 here
+     * means it never applied.
+     */
+    _cancelUncertainCharge(eventName: string, operationId: string, reservedCount: number): Promise<boolean>;
+    _voidCharge(eventName: string, operationId: string, reservedCount: number, notFoundMeansVoided?: boolean): Promise<boolean>;
+    /**
+     * @internal Retry unconfirmed voids (caller holds the operation lock).
+     * Returns false while any earlier void is still unconfirmed.
+     */
+    _retryPendingVoids(): Promise<boolean>;
     /** Clamp local reservation down when the server accepted fewer events. */
     /** @internal Used by pushDataAndCharge dual-event reconciliation. */
     applyAcceptedCount(eventName: string, reservedCount: number, acceptedCount: number, eventPrice?: number): number;
@@ -151,3 +177,9 @@ export declare function pushDataAndCharge<T>({ chargingManager, items, eventName
     isDefaultDataset: boolean;
     pushFn: (limitedItems: T | T[]) => Promise<void>;
 }): Promise<ChargeResult>;
+/**
+ * @internal True when a dataset push certainly stored nothing: the storage API
+ * answered with an error status, or the request never reached it. Timeouts and
+ * resets after the request was sent are not proof (the write may have landed).
+ */
+export declare function pushCertainlyFailed(err: unknown): boolean;

package/dist/charging.js

@@ -6,6 +6,7 @@ exports.resolveDefaultDatasetItemEventName = resolveDefaultDatasetItemEventName;
 exports.hasDefaultDatasetItemEventInPricing = hasDefaultDatasetItemEventInPricing;
 exports.mergeChargeResults = mergeChargeResults;
 exports.pushDataAndCharge = pushDataAndCharge;
+exports.pushCertainlyFailed = pushCertainlyFailed;
 const node_crypto_1 = require("node:crypto");
 /** Client safe-replay window; server TTL is longer (35d). */
 exports.CHARGE_IDEMPOTENCY_SAFE_AGE_MS = 30 * 24 * 60 * 60 * 1000;
@@ -108,6 +109,10 @@ class ChargingManager {
     chargingState;
     chargingLogDataset;
     notPpeWarningPrinted = false;
+    /** Idempotency key of the last charge the platform accepted (pushDataAndCharge void-on-failure). */
+    lastAcceptedOperationId;
+    /** Voids not yet confirmed by the platform; retried before the next push and at exit. */
+    pendingVoids = [];
     /** Unresolved platform POSTs: key = idempotency UUID (never event name). */
     pendingCharges = new Map();
     flushInFlight = null;
@@ -340,6 +345,10 @@ class ChargingManager {
         const roundedResult = Math.floor(Number(unroundedResult.toFixed(4)));
         return Math.max(0, roundedResult);
     }
+    /** @internal Remaining allowance per event (pushDataAndCharge's uncertain path). */
+    _chargeableWithinLimit() {
+        return this.chargingState === undefined ? {} : this.getChargeableWithinLimit();
+    }
     getChargeableWithinLimit() {
         if (Object.keys(this.pricingInfo).length === 0) {
             return {};
@@ -398,6 +407,76 @@ class ChargingManager {
             `${lastError?.message ?? lastError}`);
         return null;
     }
+    /** @internal Idempotency key of the charge accepted by the last _chargeExclusive call, if any. */
+    _takeLastAcceptedOperationId() {
+        const id = this.lastAcceptedOperationId;
+        this.lastAcceptedOperationId = undefined;
+        return id;
+    }
+    /**
+     * @internal Reverse an accepted charge whose items were never stored
+     * (POST /v2/actor-runs/:runId/charge/void), then unwind the local
+     * reservation. Returns true when the platform voided it.
+     */
+    /**
+     * @internal Cancel a charge whose outcome was never confirmed, for items that
+     * were then not stored: drop the pending intent (so the exit flush never
+     * posts it) and void it in case the original request did apply. A 404 here
+     * means it never applied.
+     */
+    async _cancelUncertainCharge(eventName, operationId, reservedCount) {
+        this.pendingCharges.delete(operationId);
+        return this._voidCharge(eventName, operationId, reservedCount, true);
+    }
+    async _voidCharge(eventName, operationId, reservedCount, notFoundMeansVoided = false) {
+        if (!this.isAtHome || !this.actorRunId)
+            return false;
+        const runClient = this.apifyClient.run(this.actorRunId);
+        for (let attempt = 0; attempt < this.postAttempts; attempt++) {
+            try {
+                await runClient.httpClient.call({
+                    url: runClient._url('charge/void'),
+                    method: 'POST',
+                    headers: { 'idempotency-key': operationId },
+                });
+                // Confirmed (voided or already voided): only now give the budget back.
+                this.applyAcceptedCount(eventName, reservedCount, 0);
+                return true;
+            }
+            catch (err) {
+                const status = httpStatusOf(err);
+                // Only the void route's own answer proves it never applied; a server
+                // without the route (older deployment) also answers 404.
+                if (status === 404 && notFoundMeansVoided && apiErrorTypeOf(err) === 'charge-not-found') {
+                    // The unconfirmed charge never reached the ledger: nothing billed.
+                    this.applyAcceptedCount(eventName, reservedCount, 0);
+                    return true;
+                }
+                // 404: nothing to void; 409: already settled; 4xx auth: permanent.
+                // The charge stands; keep the local reservation that matches it.
+                if (status !== undefined && status >= 400 && status < 500)
+                    return false;
+                if (attempt + 1 < this.postAttempts)
+                    await new Promise((r) => setTimeout(r, 250 * (attempt + 1)));
+            }
+        }
+        // Outcome unknown: keep the reservation and retry before the next push / at exit.
+        this.pendingVoids.push({ eventName, operationId, reservedCount, notFoundMeansVoided });
+        console.error(`[Scrapely] Void of charge ${operationId} ('${eventName}') not confirmed yet; will retry`);
+        return false;
+    }
+    /**
+     * @internal Retry unconfirmed voids (caller holds the operation lock).
+     * Returns false while any earlier void is still unconfirmed.
+     */
+    async _retryPendingVoids() {
+        const pending = this.pendingVoids;
+        this.pendingVoids = [];
+        for (const v of pending) {
+            await this._voidCharge(v.eventName, v.operationId, v.reservedCount, v.notFoundMeansVoided);
+        }
+        return this.pendingVoids.length === 0;
+    }
     /** Clamp local reservation down when the server accepted fewer events. */
     /** @internal Used by pushDataAndCharge dual-event reconciliation. */
     applyAcceptedCount(eventName, reservedCount, acceptedCount, eventPrice) {
@@ -431,6 +510,7 @@ class ChargingManager {
     async _flushPendingExclusive(maxIntents = 32, deadlineMs) {
         if (!this.actorRunId)
             return 0;
+        await this._retryPendingVoids();
         if (this.flushInFlight) {
             return this.flushInFlight;
         }
@@ -584,6 +664,8 @@ class ChargingManager {
             }
             if (accepted !== null) {
                 this.pendingCharges.delete(intent.idempotencyKey);
+                if (accepted > 0)
+                    this.lastAcceptedOperationId = intent.idempotencyKey;
                 chargedCount = this.applyAcceptedCount(normalizedEventName, chargedCount, accepted, eventPricing.price);
             }
             else {
@@ -683,11 +765,22 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
         if (chargingManager.isClosed()) {
             throw new Error('ChargingManager is closed (actor is shutting down); charge rejected');
         }
+        // A void that did not go through for an earlier failed push must land
+        // before this push charges again, or one unsaved result is billed twice.
+        const voidsConfirmed = (await chargingManager._retryPendingVoids?.()) !== false;
         const { limitedItems, eventsToCharge } = chargingManager.calculatePushDataLimits({
             items,
             eventName,
             isDefaultDataset,
         });
+        // Only events sent to the platform count: the synthetic default-dataset
+        // event is local bookkeeping and never charged through the API.
+        if (!voidsConfirmed && Object.keys(eventsToCharge).some((name) => !isSyntheticEvent(name))) {
+            // The charge for an earlier, unsaved push may still stand: charging
+            // this push now could bill one result twice. Let the caller retry.
+            // Pushes that charge nothing go through.
+            throw new Error('[Scrapely] An earlier charge for a failed dataset push could not be voided yet; retry the push');
+        }
         const itemsArray = Array.isArray(items) ? items : [items];
         const allItemsTrimmed = itemsArray.length > 0 && limitedItems.length === 0;
         if (limitedItems.length === 0) {
@@ -711,12 +804,33 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
             const reservedInOp = [];
             try {
                 for (const [name, chargeCount] of ordered) {
-                    const r = await chargingManager._chargeExclusive({
-                        eventName: name,
-                        count: chargeCount,
-                    });
+                    chargingManager._takeLastAcceptedOperationId?.();
+                    let r;
+                    try {
+                        r = await chargingManager._chargeExclusive({
+                            eventName: name,
+                            count: chargeCount,
+                        });
+                    }
+                    catch (chargeErr) {
+                        if (!(chargeErr instanceof ChargeUncertainError))
+                            throw chargeErr;
+                        // The platform did not confirm the charge (5xx / timeouts after
+                        // retries). Never drop the items for that: the intent stays
+                        // pending with its idempotency key and is re-posted by the exit
+                        // flush, so the items are stored now and billed once.
+                        console.warn(`[Scrapely] Charge for '${name}' not confirmed yet (${chargeErr.operationId}); storing the items and retrying the charge later`);
+                        r = {
+                            eventChargeLimitReached: false,
+                            chargedCount: chargeErr.count,
+                            chargeableWithinLimit: chargingManager._chargeableWithinLimit?.() ?? {},
+                        };
+                        results.push(r);
+                        reservedInOp.push({ name, reserved: chargeErr.count, operationId: chargeErr.operationId, uncertain: true });
+                        continue;
+                    }
                     results.push(r);
-                    reservedInOp.push({ name, reserved: r.chargedCount });
+                    reservedInOp.push({ name, reserved: r.chargedCount, operationId: chargingManager._takeLastAcceptedOperationId?.() });
                 }
             }
             catch (err) {
@@ -741,7 +855,12 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                 const [name] = ordered[i];
                 const reserved = results[i].chargedCount;
                 if (reserved > acceptedForItems) {
-                    chargingManager.applyAcceptedCount(name, reserved, acceptedForItems);
+                    // Nothing stored and a platform charge was made: leave its local
+                    // count to the void below, which releases it only once the
+                    // platform confirms (never on a server without the void route).
+                    const platformOp = acceptedForItems <= 0 && reservedInOp[i]?.operationId;
+                    if (!platformOp)
+                        chargingManager.applyAcceptedCount(name, reserved, acceptedForItems);
                     results[i] = {
                         ...results[i],
                         chargedCount: acceptedForItems,
@@ -751,6 +870,20 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
             }
             chargeResult = results.reduce(mergeChargeResults);
             if (acceptedForItems <= 0) {
+                // Nothing will be stored (another event of this push was capped to
+                // zero): undo every platform charge this operation made, so no
+                // result is billed for items that are never saved.
+                for (let i = 0; i < reservedInOp.length; i++) {
+                    const { name, operationId, uncertain, reserved } = reservedInOp[i];
+                    if (reserved <= 0 || !operationId)
+                        continue;
+                    // Original count: the local reservation is released only when
+                    // the platform confirms the void (or that it never applied).
+                    if (uncertain)
+                        await chargingManager._cancelUncertainCharge?.(name, operationId, reserved);
+                    else
+                        await chargingManager._voidCharge(name, operationId, reserved);
+                }
                 return {
                     eventChargeLimitReached: true,
                     chargedCount: chargeResult.chargedCount,
@@ -762,11 +895,34 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                 await pushFn(Array.isArray(items) ? toPersist : toPersist[0]);
             }
             catch (pushErr) {
-                // Charge already accepted server-side; storage failed. Do not unwind
-                // server ledger (no reverse API). Surface error so caller can retry
-                // storage; local counters already match accepted charges.
+                // Storage failed after the charge was accepted: a result is billed
+                // only if it was saved, so void this operation's platform charges
+                // and unwind the local (synthetic) reservations, then surface the
+                // error so the caller can retry the push (which charges again).
                 console.error(`[Scrapely] Dataset push failed after charge accepted ` +
-                    `(items=${acceptedForItems}): ${pushErr.message}`);
+                    `(items=${acceptedForItems}): ${pushErr.message}; voiding the charge`);
+                // Only when the push certainly stored nothing: the storage API
+                // answered with an error, or the connection was never made. A
+                // timeout or reset after sending may have stored the items, so
+                // the charge stands for those (they may be in the dataset).
+                if (!pushCertainlyFailed(pushErr))
+                    throw pushErr;
+                for (let i = 0; i < reservedInOp.length; i++) {
+                    const { name, operationId, uncertain } = reservedInOp[i];
+                    const reserved = results[i].chargedCount;
+                    if (reserved <= 0)
+                        continue;
+                    if (operationId && uncertain) {
+                        // Never re-post a charge for items that were not stored.
+                        await chargingManager._cancelUncertainCharge?.(name, operationId, reserved);
+                    }
+                    else if (operationId) {
+                        await chargingManager._voidCharge(name, operationId, reserved);
+                    }
+                    else if (isSyntheticEvent(name)) {
+                        chargingManager.applyAcceptedCount(name, reserved, 0);
+                    }
+                }
                 throw pushErr;
             }
             return chargeResult;
@@ -780,3 +936,32 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
         };
     });
 }
+/** HTTP status of an API/client error, if the server answered. */
+function httpStatusOf(err) {
+    const e = err;
+    const raw = e?.statusCode ?? e?.status ?? e?.response?.status ?? e?.response?.statusCode;
+    return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined;
+}
+const NOT_SENT_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH', 'ENETUNREACH']);
+/**
+ * @internal True when a dataset push certainly stored nothing: the storage API
+ * answered with an error status, or the request never reached it. Timeouts and
+ * resets after the request was sent are not proof (the write may have landed).
+ */
+function pushCertainlyFailed(err) {
+    for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) {
+        const status = httpStatusOf(e);
+        if (status !== undefined)
+            return status >= 400;
+        const code = e.code;
+        if (typeof code === 'string' && NOT_SENT_CODES.has(code))
+            return true;
+    }
+    return false;
+}
+/** API error type (e.g. 'charge-not-found') of a client error, if the server sent one. */
+function apiErrorTypeOf(err) {
+    const e = err;
+    const t = e?.type ?? e?.data?.error?.type ?? e?.response?.data?.error?.type;
+    return typeof t === 'string' ? t : undefined;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.8",
+    "version": "1.2.10",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -19,6 +19,7 @@
     "scripts": {
         "test": "npx tsx tests/test-mvp.ts",
         "test:charging": "npx tsx --test tests/charging-pending.test.ts",
+        "test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
         "test:charging-client": "npx tsx --test tests/charging-client.test.ts",
         "test:proxy": "npx tsx --test tests/proxy-configuration.test.ts",
         "test:browser-pool": "npx tsx --test tests/patch-browser-pool.test.ts",
@@ -30,6 +31,7 @@
         "prod:update": "bash scripts/deploy-prod.sh --update-only",
         "migrate": "npx tsx scripts/run-migration.ts",
         "test:local-run": "npx tsx --test tests/local-run.test.ts",
+        "test:input-from-kv": "npx tsx --test tests/input-from-kv.test.ts",
         "test:init-rollback": "npx tsx --test tests/init-rollback.test.ts",
         "test:apify-parity": "npx tsx --test tests/apify-parity.test.ts",
         "test:run-control": "npx tsx --test tests/run-control.test.ts",