scrapely 1.2.33 → 1.2.34

package/dist/charging.js

@@ -8,6 +8,7 @@ exports.isDefaultDatasetItemEvent = isDefaultDatasetItemEvent;
 exports.resolveDefaultDatasetItemEventName = resolveDefaultDatasetItemEventName;
 exports.hasDefaultDatasetItemEventInPricing = hasDefaultDatasetItemEventInPricing;
 exports.mergeChargeResults = mergeChargeResults;
+exports.assertItemsSerializable = assertItemsSerializable;
 exports.pushDataAndCharge = pushDataAndCharge;
 exports.validateRowCharges = validateRowCharges;
 exports.pushDataWithCharges = pushDataWithCharges;
@@ -48,7 +49,7 @@ class ChargeUncertainError extends Error {
     }
 }
 exports.ChargeUncertainError = ChargeUncertainError;
-/** Apify-compatible synthetic event for default-dataset items in pay-per-event actors. */
+/** Synthetic event for default-dataset items in pay-per-event actors. */
 /** Definitive platform rejection (402/400/409). Do not retry with the same key. */
 class ChargeRejectedError extends Error {
     operationId;
@@ -108,7 +109,7 @@ function refuseDefaultDatasetPush(itemCount) {
 exports.APIFY_DEFAULT_DATASET_ITEM_EVENT = 'apify-default-dataset-item';
 /** Scrapely-native synthetic event for default-dataset items. */
 exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = 'scrapely-default-dataset-item';
-/** Apify-compatible alias — imported actors expect this constant name. */
+/** Alias — imported actors expect this constant name. */
 exports.DEFAULT_DATASET_ITEM_EVENT = exports.APIFY_DEFAULT_DATASET_ITEM_EVENT;
 function isDefaultDatasetItemEvent(eventName) {
     return eventName === exports.APIFY_DEFAULT_DATASET_ITEM_EVENT || eventName === exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT;
@@ -128,7 +129,7 @@ function hasDefaultDatasetItemEventInPricing(perEventPrices) {
 }
 /**
  * Events that must never be POSTed to the charge API (server derives them at settle).
- * Explicit names only — a prefix heuristic would drop real priced events like `apify-foo`.
+ * Explicit names only — a prefix heuristic would drop real priced events.
  */
 function isSyntheticEvent(eventName) {
     return isDefaultDatasetItemEvent(eventName);
@@ -361,8 +362,23 @@ class ChargingManager {
     lastAcceptedOperationId;
     /** Voids not yet confirmed by the platform; retried before the next push and at exit. */
     pendingVoids = [];
+    /**
+     * @internal Platform charges for results that were not stored and that the
+     * platform definitively refused to void (event -> count). They stay billed,
+     * so the next push of that event uses them instead of charging again.
+     */
+    _voidRefusedCredit = {};
     /** Unresolved platform POSTs: key = idempotency UUID (never event name). */
     pendingCharges = new Map();
+    /**
+     * Record in the run's default key-value store that mirrors pendingCharges,
+     * so a container crash does not lose them: settlement re-applies them under
+     * their own idempotency keys. One record per container, so a restarted
+     * container never overwrites its predecessor's.
+     */
+    pendingRecordKey = `SCRAPELY_PENDING_CHARGES-${(0, node_crypto_1.randomUUID)().replace(/-/g, '').slice(0, 12)}`;
+    /** Intent keys last written to that record ('' = nothing written / empty). */
+    persistedPendingSig = '';
     flushInFlight = null;
     operationTail = Promise.resolve();
     postAttempts = POST_ATTEMPTS;
@@ -735,6 +751,39 @@ class ChargingManager {
         this.lastAcceptedOperationId = undefined;
         return id;
     }
+    /**
+     * @internal Mirror the unconfirmed charges to the run's default key-value
+     * store (best effort, only when the set changed since the last write).
+     */
+    async _persistPendingCharges() {
+        if (!this.isAtHome || !this.actorRunId)
+            return;
+        const intents = [...this.pendingCharges.values()].map(({ idempotencyKey, eventName, count, createdAt }) => ({ idempotencyKey, eventName, count, createdAt }));
+        const sig = intents.map((i) => i.idempotencyKey).sort().join(',');
+        if (sig === this.persistedPendingSig)
+            return;
+        const storeId = process.env.ACTOR_DEFAULT_KEY_VALUE_STORE_ID
+            || process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID
+            || this.configuration.get('defaultKeyValueStoreId');
+        if (!storeId || storeId === 'default')
+            return;
+        let timer;
+        try {
+            const store = this.apifyClient.keyValueStore(storeId);
+            await Promise.race([
+                store.setRecord({ key: this.pendingRecordKey, value: { v: 1, intents } }),
+                new Promise((_, reject) => { timer = setTimeout(() => reject(new Error('timed out')), 15_000); }),
+            ]);
+            this.persistedPendingSig = sig;
+        }
+        catch (err) {
+            console.warn(`[Scrapely] Could not persist ${intents.length} unconfirmed charge(s): ${err.message}`);
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+    }
     /**
      * @internal Reverse an accepted charge whose items were never stored
      * (POST /v2/actor-runs/:runId/charge/void), then unwind the local
@@ -748,7 +797,54 @@ class ChargingManager {
      */
     async _cancelUncertainCharge(eventName, operationId, reservedCount) {
         this.pendingCharges.delete(operationId);
-        return this._voidCharge(eventName, operationId, reservedCount, true);
+        const voided = await this._voidCharge(eventName, operationId, reservedCount, true);
+        // A persisted copy is harmless either way (the void leaves a tombstone),
+        // but keep the record small.
+        await this._persistPendingCharges();
+        return voided;
+    }
+    /**
+     * @internal Void (or cancel, when unconfirmed) a whole charge and say what
+     * happened: 'voided' (confirmed, local reservation released), 'stands' (a
+     * definitive answer that it cannot be voided) or 'unknown' (queued for retry,
+     * reservation kept).
+     */
+    async _voidOutcome(eventName, operationId, reservedCount, uncertain) {
+        const queued = this.pendingVoids.length;
+        const ok = uncertain
+            ? await this._cancelUncertainCharge(eventName, operationId, reservedCount)
+            : await this._voidCharge(eventName, operationId, reservedCount);
+        if (ok)
+            return 'voided';
+        return this.pendingVoids.length > queued ? 'unknown' : 'stands';
+    }
+    /** @internal A charge for `count` unstored results stands (void refused): keep it as credit. */
+    _noteUnvoidable(eventName, count) {
+        if (count <= 0)
+            return;
+        this._voidRefusedCredit[eventName] = (this._voidRefusedCredit[eventName] ?? 0) + count;
+        console.warn(`[Scrapely] Charge for ${count} unstored '${eventName}' result(s) could not be voided; ` +
+            `the next push of '${eventName}' uses it instead of charging again`);
+    }
+    /** @internal Use credit for a whole charge of `count` events (all or nothing); returns the count used. */
+    _takeVoidCredit(eventName, count) {
+        const have = this._voidRefusedCredit[eventName] ?? 0;
+        if (count <= 0 || have < count)
+            return 0;
+        this._voidRefusedCredit[eventName] = have - count;
+        return count;
+    }
+    /** @internal */
+    _returnVoidCredit(eventName, count) {
+        if (count > 0)
+            this._voidRefusedCredit[eventName] = (this._voidRefusedCredit[eventName] ?? 0) + count;
+    }
+    /** @internal Void (or cancel) a charge for results that were not stored; a refused void becomes credit. */
+    async _voidOrCredit(eventName, operationId, reservedCount, uncertain) {
+        const outcome = await this._voidOutcome(eventName, operationId, reservedCount, uncertain);
+        if (outcome === 'stands')
+            this._noteUnvoidable(eventName, reservedCount);
+        return outcome;
     }
     async _voidCharge(eventName, operationId, reservedCount, notFoundMeansVoided = false, deadline) {
         if (!this.isAtHome || !this.actorRunId)
@@ -897,6 +993,7 @@ class ChargingManager {
                 if (confirmed === 0)
                     break;
             }
+            await this._persistPendingCharges();
             return confirmedTotal;
         })();
         try {
@@ -1020,6 +1117,9 @@ class ChargingManager {
                 this.syncPlatformBudget();
             }
             else {
+                // Not confirmed: keep a durable copy so a crash before the next
+                // flush cannot lose it (settlement re-applies it).
+                await this._persistPendingCharges();
                 throw new ChargeUncertainError(`Charge for event '${normalizedEventName}' (count=${chargedCount}) was not confirmed by the platform; operationId=${intent.idempotencyKey}`, {
                     operationId: intent.idempotencyKey,
                     eventName: normalizedEventName,
@@ -1064,7 +1164,7 @@ function extractChargeResponseBody(response) {
     if (!response || typeof response !== 'object')
         return {};
     const r = response;
-    // apify-client httpClient.call returns { data, statusCode, ... }
+    // The API client's httpClient.call returns { data, statusCode, ... }
     if (r.data && typeof r.data === 'object' && !Array.isArray(r.data)) {
         return r.data;
     }
@@ -1099,6 +1199,12 @@ function classifyChargeHttpError(err, intent) {
             reason: status === 402 ? 'charge-limit-exceeded' : `http-${status}`,
         });
     }
+    // A 5xx / 408 / 429 answer is never a definitive rejection, whatever its text
+    // says: the charge may have applied, and only a retry with the same
+    // Idempotency-Key can tell (classifying it as rejected unwound the local
+    // count and the caller charged again under a new key).
+    if (status !== undefined && isTransientStatus(status))
+        return null;
     const message = String(err?.message ?? err ?? '');
     if (/charge-limit-exceeded|unknown-event|ledger-sealed|invalid-input|conflict/i.test(message)) {
         return new ChargeRejectedError(message, {
@@ -1111,7 +1217,39 @@ function classifyChargeHttpError(err, intent) {
     }
     return null;
 }
-async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultDataset, pushFn, }) {
+/**
+ * @internal Fail before anything is charged when an item cannot be serialized
+ * for the dataset (circular reference, BigInt, a bare function or undefined):
+ * the push would otherwise fail after the charge, with nothing stored.
+ */
+function assertItemsSerializable(items) {
+    items.forEach((item, index) => {
+        let json;
+        try {
+            json = JSON.stringify(item);
+        }
+        catch (err) {
+            throw new Error(`Dataset item at index ${index} is not serializable to JSON: ${err.message}`);
+        }
+        if (json === undefined)
+            throw new Error(`Dataset item at index ${index} is not serializable to JSON`);
+    });
+}
+// Credit / void helpers that also work with partial managers (tests, older shims).
+const takeVoidCredit = (m, name, count) => isSyntheticEvent(name) ? 0 : (m._takeVoidCredit?.(name, count) ?? 0);
+const returnVoidCredit = (m, name, count) => m._returnVoidCredit?.(name, count);
+async function voidOrCredit(m, name, operationId, reserved, uncertain, creditIfStands = true) {
+    if (creditIfStands && m._voidOrCredit)
+        return m._voidOrCredit(name, operationId, reserved, uncertain);
+    if (m._voidOutcome)
+        return m._voidOutcome(name, operationId, reserved, uncertain);
+    if (uncertain)
+        await m._cancelUncertainCharge?.(name, operationId, reserved);
+    else
+        await m._voidCharge(name, operationId, reserved);
+    return 'voided';
+}
+async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultDataset, pushFn, validateItems, }) {
     return chargingManager._withOperationLock(async () => {
         if (chargingManager.isClosed()) {
             throw new Error('ChargingManager is closed (actor is shutting down); charge rejected');
@@ -1178,6 +1316,9 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
         // items in the dataset. Operation lock is held across both steps.
         let chargeResult;
         if (Object.keys(eventsToCharge).length > 0) {
+            // An item the storage would refuse fails here, before it is charged.
+            assertItemsSerializable(limitedItems);
+            validateItems?.(limitedItems);
             const results = [];
             // Synthetic/default-dataset events first (local-only); then explicit.
             const ordered = Object.entries(eventsToCharge).sort(([a], [b]) => {
@@ -1185,10 +1326,35 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                 const bSyn = isSyntheticEvent(b) ? 0 : 1;
                 return aSyn - bSyn;
             });
+            // credit: events of this push paid from credit (an earlier charge that could not be voided).
             const reservedInOp = [];
+            // Undo charge i of this operation: void a platform charge, unwind a local
+            // one, give credit back. A refused void becomes credit (its results were
+            // not stored) unless the caller handles that itself (creditIfStands=false).
+            const undoOp = async (i, creditIfStands = true) => {
+                const entry = reservedInOp[i];
+                const { name, reserved, operationId, uncertain } = entry;
+                if (reserved <= 0)
+                    return 'voided';
+                if (entry.credit) {
+                    returnVoidCredit(chargingManager, name, entry.credit);
+                    entry.credit = 0;
+                    return 'voided';
+                }
+                if (operationId)
+                    return voidOrCredit(chargingManager, name, operationId, reserved, !!uncertain, creditIfStands);
+                chargingManager.applyAcceptedCount(name, reserved, 0);
+                return 'voided';
+            };
             try {
                 for (const [name, chargeCount] of ordered) {
                     chargingManager._takeLastAcceptedOperationId?.();
+                    const credit = takeVoidCredit(chargingManager, name, chargeCount);
+                    if (credit > 0) {
+                        results.push({ eventChargeLimitReached: false, chargedCount: credit, chargeableWithinLimit: chargingManager._chargeableWithinLimit?.() ?? {} });
+                        reservedInOp.push({ name, reserved: credit, credit });
+                        continue;
+                    }
                     let r;
                     try {
                         r = await chargingManager._chargeExclusive({
@@ -1218,14 +1384,64 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                 }
             }
             catch (err) {
-                // Uncertain/failure after partial multi-event reserve: unwind all
-                // prior reservations for this operation (synthetic + explicit).
-                for (const { name, reserved } of reservedInOp) {
-                    if (reserved > 0) {
-                        chargingManager.applyAcceptedCount(name, reserved, 0);
+                // Failure after a partial multi-event reserve: undo every prior
+                // charge of this operation, voiding the ones the platform accepted
+                // (not only unwinding them locally).
+                for (let i = 0; i < reservedInOp.length; i++)
+                    await undoOp(i);
+                throw err;
+            }
+            // One platform event accepted more than another: only the smaller
+            // count of items is stored, so the larger platform charge is voided
+            // and charged again for exactly that count. A charge that cannot be
+            // voided (definitive answer) stays as before; one whose void is not
+            // confirmed yet fails the push so the caller retries it.
+            // (Today a push posts at most one platform event; this keeps it
+            // correct if that changes.)
+            const stood = new Set();
+            for (let round = 0; round < 3; round++) {
+                const acceptedNow = Math.min(limitedItems.length, ...results.map((r) => r.chargedCount));
+                const excess = results
+                    .map((r, i) => i)
+                    .filter((i) => reservedInOp[i]?.operationId && !stood.has(i) && results[i].chargedCount > acceptedNow);
+                if (acceptedNow <= 0 || excess.length === 0)
+                    break;
+                for (const i of excess) {
+                    const name = reservedInOp[i].name;
+                    // A refused void here keeps its stored share; the rest becomes credit below.
+                    const outcome = await undoOp(i, false);
+                    if (outcome === 'stands') {
+                        stood.add(i);
+                        continue;
+                    }
+                    if (outcome === 'unknown') {
+                        for (let j = 0; j < reservedInOp.length; j++)
+                            if (j !== i)
+                                await undoOp(j);
+                        throw new Error('[Scrapely] An earlier charge for a failed dataset push could not be voided yet; retry the push');
+                    }
+                    chargingManager._takeLastAcceptedOperationId?.();
+                    let r;
+                    let operationId;
+                    let uncertain = false;
+                    try {
+                        r = await chargingManager._chargeExclusive({ eventName: name, count: acceptedNow });
+                        operationId = chargingManager._takeLastAcceptedOperationId?.();
+                    }
+                    catch (chargeErr) {
+                        if (!(chargeErr instanceof ChargeUncertainError))
+                            throw chargeErr;
+                        r = {
+                            eventChargeLimitReached: false,
+                            chargedCount: chargeErr.count,
+                            chargeableWithinLimit: chargingManager._chargeableWithinLimit?.() ?? {},
+                        };
+                        operationId = chargeErr.operationId;
+                        uncertain = true;
                     }
+                    results[i] = r;
+                    reservedInOp[i] = { name, reserved: r.chargedCount, operationId, uncertain };
                 }
-                throw err;
             }
             // Persist only what every charged event accepted.
             // When multiple events are charged for the same items, use the
@@ -1243,8 +1459,22 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                     // count to the void below, which releases it only once the
                     // platform confirms (never on a server without the void route).
                     const platformOp = acceptedForItems <= 0 && reservedInOp[i]?.operationId;
-                    if (!platformOp)
-                        chargingManager.applyAcceptedCount(name, reserved, acceptedForItems);
+                    if (!platformOp) {
+                        const entry = reservedInOp[i];
+                        let over = reserved - acceptedForItems;
+                        const back = Math.min(over, entry.credit ?? 0);
+                        if (back > 0) {
+                            returnVoidCredit(chargingManager, name, back);
+                            entry.credit = (entry.credit ?? 0) - back;
+                            entry.reserved -= back;
+                            over -= back;
+                        }
+                        // The platform refused to void it: the unstored share stays billed.
+                        if (over > 0 && stood.has(i))
+                            chargingManager._noteUnvoidable?.(name, over);
+                        else if (over > 0)
+                            chargingManager.applyAcceptedCount(name, acceptedForItems + over, acceptedForItems);
+                    }
                     results[i] = {
                         ...results[i],
                         chargedCount: acceptedForItems,
@@ -1252,7 +1482,7 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                     };
                 }
             }
-            // Report rows charged, as Apify does for the pushed event: summing the
+            // Report rows charged for the pushed event: summing the
             // explicit and default-dataset events would count every row twice.
             chargeResult = { ...results.reduce(mergeChargeResults), chargedCount: acceptedForItems };
             if (isDefaultDataset && acceptedForItems < limitedItems.length) {
@@ -1263,15 +1493,19 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                 // zero): undo every platform charge this operation made, so no
                 // result is billed for items that are never saved.
                 for (let i = 0; i < reservedInOp.length; i++) {
-                    const { name, operationId, uncertain, reserved } = reservedInOp[i];
+                    const entry = reservedInOp[i];
+                    const { name, operationId, uncertain } = entry;
+                    let { reserved } = entry;
+                    if (entry.credit) {
+                        returnVoidCredit(chargingManager, name, entry.credit);
+                        reserved -= entry.credit;
+                        entry.credit = 0;
+                    }
                     if (reserved <= 0 || !operationId)
                         continue;
                     // Original count: the local reservation is released only when
                     // the platform confirms the void (or that it never applied).
-                    if (uncertain)
-                        await chargingManager._cancelUncertainCharge?.(name, operationId, reserved);
-                    else
-                        await chargingManager._voidCharge(name, operationId, reserved);
+                    await voidOrCredit(chargingManager, name, operationId, reserved, !!uncertain);
                 }
                 return {
                     eventChargeLimitReached: true,
@@ -1297,16 +1531,21 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                 if (!pushCertainlyFailed(pushErr))
                     throw pushErr;
                 for (let i = 0; i < reservedInOp.length; i++) {
-                    const { name, operationId, uncertain } = reservedInOp[i];
-                    const reserved = results[i].chargedCount;
+                    const entry = reservedInOp[i];
+                    const { name, operationId, uncertain } = entry;
+                    let reserved = results[i].chargedCount;
+                    if (entry.credit) {
+                        const back = Math.min(reserved, entry.credit);
+                        returnVoidCredit(chargingManager, name, back);
+                        entry.credit -= back;
+                        reserved -= back;
+                    }
                     if (reserved <= 0)
                         continue;
-                    if (operationId && uncertain) {
-                        // Never re-post a charge for items that were not stored.
-                        await chargingManager._cancelUncertainCharge?.(name, operationId, reserved);
-                    }
-                    else if (operationId) {
-                        await chargingManager._voidCharge(name, operationId, reserved);
+                    if (operationId) {
+                        // An unconfirmed charge is cancelled (never re-posted for items
+                        // that were not stored); a refused void becomes credit.
+                        await voidOrCredit(chargingManager, name, operationId, reserved, !!uncertain);
                     }
                     else if (isSyntheticEvent(name)) {
                         chargingManager.applyAcceptedCount(name, reserved, 0);
@@ -1351,7 +1590,7 @@ function validateRowCharges(charges) {
  * the rows that are kept, so no row is stored part-paid and no event is billed
  * for a row that was not stored. Unpriced events are ignored.
  */
-async function pushDataWithCharges({ chargingManager, items, charges, isDefaultDataset, pushFn, chargePer = 'row', }) {
+async function pushDataWithCharges({ chargingManager, items, charges, isDefaultDataset, pushFn, chargePer = 'row', validateItems, }) {
     const perRowIn = validateRowCharges(charges);
     if (chargePer !== 'row' && chargePer !== 'batch')
         throw new Error("chargePer must be 'row' or 'batch'");
@@ -1384,16 +1623,20 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
         if (!voidsConfirmed && Object.keys(perRow).some((n) => !isSyntheticEvent(n))) {
             throw new Error('[Scrapely] An earlier charge for a failed dataset push could not be voided yet; retry the push');
         }
+        // An item the storage would refuse fails here, before it is charged.
+        assertItemsSerializable(rowsIn);
+        validateItems?.(rowsIn);
         const undo = async (ops) => {
             for (const op of ops) {
                 if (op.charged <= 0)
                     continue;
-                if (!op.operationId)
+                if (op.credit)
+                    returnVoidCredit(chargingManager, op.name, op.charged);
+                else if (!op.operationId)
                     chargingManager.applyAcceptedCount(op.name, op.charged, 0);
-                else if (op.uncertain)
-                    await chargingManager._cancelUncertainCharge(op.name, op.operationId, op.charged);
+                // Cancel / void; a refused void stays billed and becomes credit.
                 else
-                    await chargingManager._voidCharge(op.name, op.operationId, op.charged);
+                    await voidOrCredit(chargingManager, op.name, op.operationId, op.charged, op.uncertain);
             }
         };
         const order = Object.entries(perRow).sort(([a], [b]) => Number(isSyntheticEvent(a)) - Number(isSyntheticEvent(b)) || a.localeCompare(b));
@@ -1405,19 +1648,25 @@ async function pushDataWithCharges({ chargingManager, items, charges, isDefaultD
                 const want = n * rows;
                 chargingManager._takeLastAcceptedOperationId?.();
                 let got;
-                try {
-                    const r = await chargingManager._chargeExclusive({ eventName: name, count: want });
-                    got = r.chargedCount;
-                    ops.push({ name, charged: got, operationId: chargingManager._takeLastAcceptedOperationId?.(), uncertain: false });
+                if (takeVoidCredit(chargingManager, name, want) > 0) {
+                    got = want;
+                    ops.push({ name, charged: want, uncertain: false, credit: true });
                 }
-                catch (err) {
-                    if (!(err instanceof ChargeUncertainError)) {
-                        await undo(ops);
-                        throw err;
+                else {
+                    try {
+                        const r = await chargingManager._chargeExclusive({ eventName: name, count: want });
+                        got = r.chargedCount;
+                        ops.push({ name, charged: got, operationId: chargingManager._takeLastAcceptedOperationId?.(), uncertain: false });
+                    }
+                    catch (err) {
+                        if (!(err instanceof ChargeUncertainError)) {
+                            await undo(ops);
+                            throw err;
+                        }
+                        // Not confirmed: stays pending and is re-posted at exit, so it counts as accepted.
+                        got = err.count;
+                        ops.push({ name, charged: got, operationId: err.operationId, uncertain: true });
                     }
-                    // Not confirmed: stays pending and is re-posted at exit, so it counts as accepted.
-                    got = err.count;
-                    ops.push({ name, charged: got, operationId: err.operationId, uncertain: true });
                 }
                 if (got < want) {
                     shortTo = Math.floor(got / n);

package/dist/configuration.js

@@ -86,8 +86,8 @@ class Configuration extends core_1.Configuration {
         defaultDatasetId: 'default',
         defaultRequestQueueId: 'default',
         inputKey: 'INPUT',
-        apiBaseUrl: 'https://api-staging.scrape.ly',
-        apiPublicBaseUrl: 'https://api-staging.scrape.ly',
+        apiBaseUrl: 'https://api.scrape.ly',
+        apiPublicBaseUrl: 'https://api.scrape.ly',
         proxyStatusUrl: 'http://proxy.scrape.ly',
         proxyHostname: 'proxy.scrape.ly',
         proxyPort: 8000,

package/dist/index.js

@@ -90,7 +90,7 @@ Object.defineProperty(exports, "MissingRouteError", { enumerable: true, get: fun
 Object.defineProperty(exports, "CriticalError", { enumerable: true, get: function () { return core_2.CriticalError; } });
 Object.defineProperty(exports, "useState", { enumerable: true, get: function () { return core_2.useState; } });
 Object.defineProperty(exports, "mergeCookies", { enumerable: true, get: function () { return core_2.mergeCookies; } });
-// Apify v2 compatible utils namespace
+// v2-style utils namespace
 var utils_js_1 = require("./utils.js");
 Object.defineProperty(exports, "utils", { enumerable: true, get: function () { return utils_js_1.utils; } });
 Object.defineProperty(exports, "enqueueLinks", { enumerable: true, get: function () { return utils_js_1.enqueueLinks; } });

package/dist/node-wrapper.js

@@ -57,7 +57,7 @@ const SCRIPT_CONFIG_ENV_RE = new RegExp(`^npm_config_(${[...SCRIPT_CONFIG_KEYS,
 /**
  * Whether an npmrc file sets any of SCRIPT_CONFIG_KEYS. Other settings
  * (progress=false, loglevel, registry, auth, ...) don't affect running a
- * script — Apify's Dockerfiles write `progress=false` to ~/.npmrc.
+ * script — the standard actor Dockerfiles write `progress=false` to ~/.npmrc.
  * Unreadable or unparseable files count as setting them.
  */
 function readNpmrc(file) {

package/dist/patch-browser-pool.js

@@ -1033,8 +1033,12 @@ function installEsmPuppeteerHook() {
         return;
     Module._scrapelyEsmPuppeteerHook = true;
     try {
-        if (typeof Module.registerHooks === 'function') {
-            // Node 22.15+ / 23.5+: synchronous, in-thread.
+        // Node 22's synchronous hooks break require(esm) of CommonJS graphs that
+        // ESM imports (crawlee -> linkedom: "request for 'boolbase' is not in cache");
+        // Node 22 uses the off-thread loader instead.
+        const nodeMajor = Number(process.versions.node.split('.')[0]);
+        if (typeof Module.registerHooks === 'function' && nodeMajor >= 24) {
+            // Node 24+: synchronous, in-thread.
             Module.registerHooks({
                 load(url, context, nextLoad) {
                     const result = nextLoad(url, context);
@@ -1047,7 +1051,7 @@ function installEsmPuppeteerHook() {
             });
         }
         else if (typeof Module.register === 'function') {
-            // Node 20.6+ / 22.0-22.14: off-thread loader (esm-loader-hooks.mjs).
+            // Node 20.6+ / 22: off-thread loader (esm-loader-hooks.mjs).
             // eslint-disable-next-line @typescript-eslint/no-var-requires
             const { pathToFileURL } = require('url');
             Module.register('./esm-loader-hooks.mjs', pathToFileURL(__filename));

package/dist/proxy-configuration.d.ts

@@ -1,8 +1,8 @@
 import type { ProxyConfigurationOptions as CoreProxyConfigurationOptions, ProxyInfo as CoreProxyInfo } from '@crawlee/core';
 import { ProxyConfiguration } from './scrapely-proxy-configuration.js';
 /**
- * Proxy info enriched with the platform-proxy (Apify-compatible) username
- * fields. Mirrors the Apify SDK's `ProxyInfo` extension of Crawlee's type.
+ * Proxy info enriched with the platform-proxy username fields
+ * (extends Crawlee's `ProxyInfo` type).
  */
 export interface ProxyInfo extends CoreProxyInfo {
     /** Proxy groups used by the platform proxy (e.g. `['RESIDENTIAL']`). */
@@ -18,33 +18,33 @@ export interface ProxyInfo extends CoreProxyInfo {
  * Modes supported:
  *  - Platform proxy (default on Scrapely runs): when the server injects
  *    APIFY_PROXY_* or SCRAPELY_PROXY_* credentials, groups/countryCode/password
- *    build Apify-style gateway usernames automatically.
+ *    build gateway usernames automatically.
  *  - Bring-your-own-proxy (BYOP): custom proxyUrls / newUrlFunction /
  *    tieredProxyUrls always win over the platform proxy.
  *  - Mixed: the 'auto' (or 'http://auto') sentinel inside proxyUrls /
  *    tieredProxyUrls / tieredProxyConfig[].proxyUrls reserves a slot for the
- *    platform proxy alongside BYOP entries (Apify-compatible).
+ *    platform proxy alongside BYOP entries.
  */
 export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions {
     /**
-     * Multiple different ProxyConfigurationOptions stratified into tiers (Apify-compatible).
+     * Multiple different ProxyConfigurationOptions stratified into tiers.
      * Tiers with `proxyUrls` map directly to tieredProxyUrls; tiers without
      * URLs expand their platform options (groups/countryCode/password) to a
-     * concrete platform proxy URL, mirroring Apify's `_generateTieredProxyUrls`.
+     * concrete platform proxy URL.
      */
     tieredProxyConfig?: Omit<ProxyConfigurationOptions, 'tieredProxyConfig' | 'tieredProxyUrls'>[];
     /**
-     * Platform proxy groups (Apify-compatible). Effective in platform proxy
+     * Platform proxy groups. Effective in platform proxy
      * mode; ignored for BYOP configurations.
      */
     groups?: string[];
     /**
-     * Platform proxy country code (Apify-compatible). Effective in platform
+     * Platform proxy country code. Effective in platform
      * proxy mode; ignored for BYOP configurations.
      */
     countryCode?: string;
     /**
-     * Platform proxy subdivision code, e.g. US state (Apify-compatible).
+     * Platform proxy subdivision code, e.g. US state.
      * Requires `countryCode`. Effective in platform proxy mode; ignored for
      * BYOP configurations. Emitted as a merged `country-XX_YY` username
      * fragment (ISO 3166-2).
@@ -90,7 +90,7 @@ export interface CreateProxyConfigurationOptions extends ProxyConfigurationOptio
 /**
  * Platform proxy connection details injected into run containers by the
  * Scrapely server (backed by Evomi). SCRAPELY_* vars take precedence over the
- * Apify-compatible aliases so Scrapely-first deployments stay unambiguous.
+ * APIFY_* aliases so Scrapely-first deployments stay unambiguous.
  */
 export interface PlatformProxyEnv {
     hostname: string;
@@ -114,7 +114,7 @@ export declare function getPlatformProxyEnv(): PlatformProxyEnv | null;
  * Creates a proxy configuration for use with crawlers.
  *
  * The platform proxy (Evomi-backed) is injected into runs via the SCRAPELY_PROXY
- * and APIFY_PROXY environment variables. Apify Proxy options (groups, countryCode, ...)
+ * and APIFY_PROXY environment variables. Platform proxy options (groups, countryCode, ...)
  * build gateway usernames in platform mode; proxyUrls, newUrlFunction and
  * tieredProxyUrls configure bring-your-own proxies instead.
  */

package/dist/proxy-configuration.js

@@ -57,7 +57,7 @@ function normalizeProxyInput(options) {
  * Creates a proxy configuration for use with crawlers.
  *
  * The platform proxy (Evomi-backed) is injected into runs via the SCRAPELY_PROXY
- * and APIFY_PROXY environment variables. Apify Proxy options (groups, countryCode, ...)
+ * and APIFY_PROXY environment variables. Platform proxy options (groups, countryCode, ...)
  * build gateway usernames in platform mode; proxyUrls, newUrlFunction and
  * tieredProxyUrls configure bring-your-own proxies instead.
  */

package/dist/proxy-input-aliases.d.ts

@@ -1,7 +1,7 @@
 /**
  * Proxy input objects with both namings. An input-schema proxy field may be
  * written with the Scrapely names (`useScrapelyProxy`, `scrapelyProxyGroups`,
- * `scrapelyProxyCountry`, `scrapelyProxySubdivision`) or the Apify names
+ * `scrapelyProxyCountry`, `scrapelyProxySubdivision`) or the legacy names
  * (`useApifyProxy`, `apifyProxy*`). Actor.getInput() and Actor.input return
  * every set option under both names, so actor code that reads either naming
  * sees the user's choice.