scrapely 1.2.37 → 1.2.38

package/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [1.2.38] - 2026-10-27
+
+- A key-value store record write (`Actor.setValue`, `KeyValueStore.setValue`) is no longer lost to one failed answer. Each write now sends an `Idempotency-Key`, and when its answer is lost (a `502`/`504` page from the edge or load balancer, a dropped connection, a timeout) it is sent again with the same key; the API applies and counts it once. Before, the write was never re-sent, so one such page on the final `OUTPUT` write failed the whole run. An API that answers a keyed write without `X-Scrapely-Idempotency` (an older server) gets no re-sends, as before.
+
 ## [1.2.37] - 2026-10-26
 
 - Opening a named dataset, key-value store or request queue (`Actor.openDataset('name')` and the like) retries when the API answers `503 storage-name-busy`. The API sends it when another request kept creating or renaming a storage of that name for over 15 s; nothing was changed, so the open is simply repeated. It used to fail the open.

package/dist/storage/api-storage-client.d.ts

@@ -26,7 +26,14 @@ export declare class ApiStorageClientError extends Error {
 export declare function isRetryableResponse(status: number, idempotent: boolean, errorType?: string, bodyText?: string): boolean;
 /** Default time budget for retrying one API call (429, 503, network errors). */
 export declare const DEFAULT_API_RETRY_WINDOW_MS = 300000;
-/** Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes. */
+/** A key-value store record URL (`.../key-value-stores/<id>/records/<key>`). */
+export declare function isKvRecordUrl(url: string): boolean;
+/**
+ * Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes.
+ * Record PUTs count as de-duplicated until a keyed record PUT was stored without the support
+ * header: re-applying the same body to the same record leaves the same record, so the worst
+ * case on an API that ignores the key is one extra counted write, while not re-sending fails the run.
+ */
 export declare function apiDedupesKeyedWrites(url?: string): boolean;
 /** Test hook: forget what was learned about the API, or force an answer for every origin. */
 export declare function _resetKeyedWriteSupport(value?: boolean): void;

package/dist/storage/api-storage-client.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.BaseApiClient = exports.DEFAULT_API_RETRY_WINDOW_MS = exports.ApiStorageClientError = void 0;
 exports.isRetryableResponse = isRetryableResponse;
+exports.isKvRecordUrl = isKvRecordUrl;
 exports.apiDedupesKeyedWrites = apiDedupesKeyedWrites;
 exports._resetKeyedWriteSupport = _resetKeyedWriteSupport;
 exports.apiRetryWindowMs = apiRetryWindowMs;
@@ -139,7 +140,9 @@ function isApiStorageClientError(err) {
     return err instanceof ApiStorageClientError;
 }
 // PUT is left out: a KV record PUT is counted (and billed) as a write each
-// time it is applied, so a lost response must not be re-sent blindly.
+// time it is applied, so a lost response must not be re-sent blindly. Record
+// PUTs carry an Idempotency-Key instead (see isKvRecordUrl), which the API
+// de-duplicates, so they are re-sent after a lost answer (e.g. an edge 502).
 const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'DELETE', 'OPTIONS']);
 // HAProxy's own 503 page: no backend was available, so nothing was forwarded.
 const NO_BACKEND_MARKER = 'No server is available to handle this request';
@@ -180,20 +183,39 @@ const IDEMPOTENCY_SUPPORT_HEADER = 'x-scrapely-idempotency';
 const keyedWritesDeduped = new Map();
 /** Test override for every origin (undefined: learn from answers). */
 let keyedWritesDedupedOverride;
-function originOf(url) {
+/** A key-value store record URL (`.../key-value-stores/<id>/records/<key>`). */
+function isKvRecordUrl(url) {
+    let path = url;
     try {
-        return new URL(url).origin;
+        path = new URL(url).pathname;
     }
-    catch {
-        return url;
+    catch { /* a path already */ }
+    return path.includes('/key-value-stores/') && path.includes('/records/');
+}
+/** What is learned per API: its origin, separately for record PUTs and other keyed writes. */
+function originOf(url) {
+    let origin = url;
+    try {
+        origin = new URL(url).origin;
     }
+    catch { /* keep the input */ }
+    return isKvRecordUrl(url) ? `${origin}#kv-record` : origin;
 }
-/** Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes. */
+/**
+ * Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes.
+ * Record PUTs count as de-duplicated until a keyed record PUT was stored without the support
+ * header: re-applying the same body to the same record leaves the same record, so the worst
+ * case on an API that ignores the key is one extra counted write, while not re-sending fails the run.
+ */
 function apiDedupesKeyedWrites(url) {
     if (keyedWritesDedupedOverride !== undefined)
         return keyedWritesDedupedOverride;
-    if (url !== undefined)
-        return keyedWritesDeduped.get(originOf(url)) === true;
+    if (url !== undefined) {
+        const learned = keyedWritesDeduped.get(originOf(url));
+        if (learned === undefined && isKvRecordUrl(url))
+            return true;
+        return learned === true;
+    }
     return [...keyedWritesDeduped.values()].pop() === true;
 }
 /** Test hook: forget what was learned about the API, or force an answer for every origin. */
@@ -792,10 +814,13 @@ class ApiKeyValueStoreClient extends BaseApiClient {
             ct = 'application/json';
         }
         // Pass timeout/retry options from KeyValueStoreRecordOptions to the API request
-        const requestOptions = options ? {
-            timeoutSecs: options.timeoutSecs,
-            doNotRetryTimeouts: options.doNotRetryTimeouts,
-        } : undefined;
+        // One Idempotency-Key for every attempt of this write: a re-send after a lost answer
+        // (edge 502/504, dropped connection) is applied and counted once.
+        const requestOptions = {
+            timeoutSecs: options?.timeoutSecs,
+            doNotRetryTimeouts: options?.doNotRetryTimeouts,
+            idempotencyKey: (0, node_crypto_1.randomUUID)(),
+        };
         await this.request('PUT', `/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`, body, contentType || ct, requestOptions);
     }
     async deleteRecord(key) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.37",
+    "version": "1.2.38",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -75,7 +75,8 @@
         "test:charge-retries": "npx tsx --test tests/charge-retries.test.ts",
         "test:solver": "npx tsx --test tests/solver.test.ts",
         "test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts",
-        "test:v2-utils": "npx tsx --test tests/v2-utils.test.ts"
+        "test:v2-utils": "npx tsx --test tests/v2-utils.test.ts",
+        "test:default-api-url": "npx tsx --test tests/default-api-url.test.ts"
     },
     "author": "TeraShift GmbH (Scrape.ly)",
     "license": "ISC AND Apache-2.0",
@@ -92,7 +93,7 @@
         "lodash": "^4.18.1",
         "mysql2": "^3.23.1",
         "playwright": "^1.58.2",
-        "scrapely-client": "^2.25.2",
+        "scrapely-client": "^2.25.9",
         "tar-fs": "2.1.5",
         "ws": "^8.18.0"
     },