scrapely 1.2.34 → 1.2.36

package/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.2.36] - 2026-10-22
+
+- Debug captures (`Actor.debugCapture`, `Actor.debugCaptureSync`) keep payloads over 256 KiB whole. They are stored gzip-compressed and base64-encoded as `{ key, payload, encoding: 'gzip-base64', length, truncated }` instead of being cut at 262,144 characters; up to 8 Mi characters are kept. A payload that does not compress enough keeps its first 256 KiB, now marked `truncated: true` with its full `length`. `Uint8Array` / `Buffer` payloads are stored as UTF-8 text. Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).
+
+## [1.2.35] - 2026-10-20
+
+- The storage client percent-encodes request ids and key-value store record keys as one URL path segment and refuses `''`, `.` and `..`. A request id such as `..` read from a queue used to send the call to another resource.
+
 ## [1.2.34] - 2026-10-19
 
 - Initial public release.

package/dist/actor.d.ts

@@ -459,7 +459,15 @@ export declare class Actor {
      */
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
     private static _debugCaptureCount;
+    /**
+     * A payload up to DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+     * (base64, at most DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to DEBUG_CAPTURE_MAX_CHARS
+     * characters / DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+     * needed when a large API document starts with site navigation.
+     */
     static readonly DEBUG_CAPTURE_MAX_BYTES: number;
+    static readonly DEBUG_CAPTURE_MAX_CHARS: number;
+    static readonly DEBUG_CAPTURE_MAX_ENCODED: number;
     static readonly DEBUG_CAPTURE_MAX_RECORDS = 500;
     /** The key-value store record name for a capture key (item URL or id). */
     static debugCaptureRecordKey(key: string): Promise<string>;
@@ -471,6 +479,8 @@ export declare class Actor {
     /** debugCapture for synchronous code: fire and forget. Does nothing unless capture is enabled for the run. */
     static debugCaptureSync(key: string, payload: unknown): void;
     static debugCapture(key: string, payload: unknown): Promise<void>;
+    /** The record for one capture: plain when small, else gzip-base64 (plain head as the last resort). */
+    static debugCaptureRecord(key: string, text: string): Promise<Record<string, unknown>>;
     /**
      * Helper to resolve storage identifier to a string ID or name.
      * Supports: string, { id: '...' }, { name: '...' }, { alias: '...' }

package/dist/actor.js

@@ -660,7 +660,15 @@ class Actor {
         await store.setValue(key, value, options);
     }
     static _debugCaptureCount = 0;
+    /**
+     * A payload up to DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+     * (base64, at most DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to DEBUG_CAPTURE_MAX_CHARS
+     * characters / DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+     * needed when a large API document starts with site navigation.
+     */
     static DEBUG_CAPTURE_MAX_BYTES = 256 * 1024;
+    static DEBUG_CAPTURE_MAX_CHARS = 8 * 1024 * 1024;
+    static DEBUG_CAPTURE_MAX_ENCODED = 8 * 1024 * 1024;
     static DEBUG_CAPTURE_MAX_RECORDS = 500;
     /** The key-value store record name for a capture key (item URL or id). */
     static async debugCaptureRecordKey(key) {
@@ -684,14 +692,32 @@ class Actor {
         if (Actor._debugCaptureCount >= Actor.DEBUG_CAPTURE_MAX_RECORDS)
             return;
         try {
-            const text = (typeof payload === 'string' ? payload : JSON.stringify(payload) ?? '').slice(0, Actor.DEBUG_CAPTURE_MAX_BYTES);
+            const text = typeof payload === 'string' ? payload
+                : payload instanceof Uint8Array ? Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength).toString('utf8')
+                    : JSON.stringify(payload) ?? '';
+            const record = await Actor.debugCaptureRecord(String(key), text);
             Actor._debugCaptureCount++;
-            await Actor.setValue(await Actor.debugCaptureRecordKey(key), { key: String(key), payload: text });
+            await Actor.setValue(await Actor.debugCaptureRecordKey(key), record);
         }
         catch {
             // never break the run
         }
     }
+    /** The record for one capture: plain when small, else gzip-base64 (plain head as the last resort). */
+    static async debugCaptureRecord(key, text) {
+        const limit = Actor.DEBUG_CAPTURE_MAX_BYTES;
+        if (text.length <= limit)
+            return { key, payload: text };
+        const { gzipSync } = await import('node:zlib');
+        const raw = Buffer.from(text.slice(0, Actor.DEBUG_CAPTURE_MAX_CHARS), 'utf8');
+        const kept = raw.subarray(0, Actor.DEBUG_CAPTURE_MAX_ENCODED);
+        const packed = gzipSync(kept, { level: 6 }).toString('base64');
+        if (packed.length <= limit) {
+            const truncated = text.length > Actor.DEBUG_CAPTURE_MAX_CHARS || kept.length < raw.length;
+            return { key, payload: packed, encoding: 'gzip-base64', length: text.length, truncated };
+        }
+        return { key, payload: text.slice(0, limit), length: text.length, truncated: true };
+    }
     /**
      * Helper to resolve storage identifier to a string ID or name.
      * Supports: string, { id: '...' }, { name: '...' }, { alias: '...' }
@@ -1330,12 +1356,10 @@ class Actor {
         if (!apiUrl || !token || !runId) {
             throw new Error('Metamorph requires apiBaseUrl, token, and actorRunId in configuration');
         }
-        // Wire format (server contract, Python SDK parity): the server copies the
-        // current INPUT to INPUT-METAMORPH-<n> itself and tracks the metamorph
-        // count in run stats; the new input (if any) is read from the request
-        // body's `input` field and written to INPUT. `inputKey` is NOT part of
-        // the contract — the old client-side KVS write made the server overwrite
-        // our value with the stale INPUT, silently losing the new input.
+        // Wire format (server contract, Python SDK parity): the new input (if
+        // any) goes in the request body's `input` field; the server stores it as
+        // INPUT-METAMORPH-<n> (INPUT keeps the original) and tracks the metamorph
+        // count in run stats. `inputKey` is NOT part of the contract.
         const body = { targetActorId };
         if (options?.build) {
             body.build = options.build;

package/dist/storage/api-storage-client.js

@@ -11,6 +11,7 @@ exports.fetchWithRetries = fetchWithRetries;
 const node_crypto_1 = require("node:crypto");
 const charging_js_1 = require("../charging.js");
 const kvs_iteration_js_1 = require("./kvs-iteration.js");
+const utils_js_1 = require("./utils.js");
 function buildRqQueryParams(baseClientKey, options) {
     const params = new URLSearchParams();
     params.set('clientKey', options?.clientKey ?? baseClientKey);
@@ -734,11 +735,11 @@ class ApiKeyValueStoreClient extends BaseApiClient {
         await this.request('DELETE', `/key-value-stores/${this.id}`);
     }
     async recordExists(key) {
-        return this.exists(`/key-value-stores/${this.id}/records/${key}`);
+        return this.exists(`/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`);
     }
     async getRecord(key, options) {
         try {
-            const { data, contentType } = await this.requestWithHeaders('GET', `/key-value-stores/${this.id}/records/${key}`);
+            const { data, contentType } = await this.requestWithHeaders('GET', `/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`);
             if (data === undefined || data === null)
                 return undefined;
             let value = data;
@@ -791,10 +792,10 @@ class ApiKeyValueStoreClient extends BaseApiClient {
             timeoutSecs: options.timeoutSecs,
             doNotRetryTimeouts: options.doNotRetryTimeouts,
         } : undefined;
-        await this.request('PUT', `/key-value-stores/${this.id}/records/${key}`, body, contentType || ct, requestOptions);
+        await this.request('PUT', `/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`, body, contentType || ct, requestOptions);
     }
     async deleteRecord(key) {
-        await this.request('DELETE', `/key-value-stores/${this.id}/records/${key}`);
+        await this.request('DELETE', `/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`);
     }
     async listKeys(options) {
         const params = new URLSearchParams();
@@ -853,7 +854,7 @@ class ApiKeyValueStoreClient extends BaseApiClient {
         // apiUrl is the container-internal API address; links handed to users
         // must use the public base URL the platform injects.
         const publicBase = (process.env.APIFY_API_PUBLIC_BASE_URL || process.env.ACTOR_API_PUBLIC_BASE_URL || this.config.apiUrl).replace(/\/+$/, '');
-        return `${publicBase}/v2/key-value-stores/${this.id}/records/${key}`;
+        return `${publicBase}/v2/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`;
     }
 }
 exports.ApiKeyValueStoreClient = ApiKeyValueStoreClient;
@@ -920,7 +921,7 @@ class ApiRequestQueueClient extends BaseApiClient {
     }
     async getRequest(id) {
         try {
-            return await this.request('GET', `/request-queues/${this.id}/requests/${id}`);
+            return await this.request('GET', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(id)}`);
         }
         catch {
             return undefined;
@@ -928,10 +929,10 @@ class ApiRequestQueueClient extends BaseApiClient {
     }
     async updateRequest(request, options) {
         const query = buildRqQueryParams(this.clientKey, { forefront: options?.forefront });
-        return this.request('PUT', `/request-queues/${this.id}/requests/${request.id}${query}`, request);
+        return this.request('PUT', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(request.id)}${query}`, request);
     }
     async deleteRequest(id) {
-        await this.request('DELETE', `/request-queues/${this.id}/requests/${id}`);
+        await this.request('DELETE', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(id)}`);
     }
     async listHead(options) {
         const query = buildRqQueryParams(this.clientKey, { limit: options?.limit });
@@ -959,12 +960,12 @@ class ApiRequestQueueClient extends BaseApiClient {
             lockSecs: options.lockSecs,
             forefront: options.forefront,
         });
-        const result = await this.request('PUT', `/request-queues/${this.id}/requests/${id}/lock${query}`);
+        const result = await this.request('PUT', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(id)}/lock${query}`);
         return { lockExpiresAt: new Date(result.lockExpiresAt) };
     }
     async deleteRequestLock(id, options) {
         const query = buildRqQueryParams(this.clientKey, { forefront: options?.forefront });
-        await this.request('DELETE', `/request-queues/${this.id}/requests/${id}/lock${query}`);
+        await this.request('DELETE', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(id)}/lock${query}`);
     }
     /**
      * List requests in the queue with optional filtering and pagination.

package/dist/storage/key-value-store-client.js

@@ -297,7 +297,7 @@ class DragonflyKeyValueStoreClient {
     getPublicUrl(key) {
         const config = configuration_js_1.Configuration.getGlobalConfig();
         const baseUrl = config.get('apiPublicBaseUrl') || config.get('apiBaseUrl') || 'https://api.scrape.ly';
-        return `${baseUrl}/v2/key-value-stores/${this.id}/records/${key}`;
+        return `${baseUrl}/v2/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`;
     }
 }
 exports.DragonflyKeyValueStoreClient = DragonflyKeyValueStoreClient;

package/dist/storage/utils.d.ts

@@ -1,3 +1,11 @@
 export declare function uniqueKeyToRequestId(uniqueKey: string): string;
 export declare function generateId(): string;
 export declare function nowISO(): Date;
+/**
+ * Percent-encode a value so it stays one URL path segment. Request ids come
+ * from queue contents another client may have written: unencoded, an id such
+ * as `..` or `../../key-value-stores/x` would send the call to another
+ * resource. `''`, `.` and `..` are rejected because a URL parser resolves dot
+ * segments even when they are percent-encoded.
+ */
+export declare function pathSegment(value: string): string;

package/dist/storage/utils.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.uniqueKeyToRequestId = uniqueKeyToRequestId;
 exports.generateId = generateId;
 exports.nowISO = nowISO;
+exports.pathSegment = pathSegment;
 const node_crypto_1 = require("node:crypto");
 function uniqueKeyToRequestId(uniqueKey) {
     return (0, node_crypto_1.createHash)('sha256')
@@ -17,3 +18,19 @@ function generateId() {
 function nowISO() {
     return new Date();
 }
+/**
+ * Percent-encode a value so it stays one URL path segment. Request ids come
+ * from queue contents another client may have written: unencoded, an id such
+ * as `..` or `../../key-value-stores/x` would send the call to another
+ * resource. `''`, `.` and `..` are rejected because a URL parser resolves dot
+ * segments even when they are percent-encoded.
+ */
+function pathSegment(value) {
+    if (typeof value !== 'string') {
+        throw new TypeError(`Expected a string for a URL path segment, got ${typeof value}.`);
+    }
+    if (value === '' || value === '.' || value === '..') {
+        throw new Error(`"${value}" cannot be used as a URL path segment.`);
+    }
+    return encodeURIComponent(value);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.34",
+    "version": "1.2.36",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -47,6 +47,8 @@
         "test:charge-pending-persist": "npx tsx --test tests/charge-pending-persist.test.ts",
         "test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
         "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
+        "test:storage-path-segments": "npx tsx --test tests/storage-path-segments.test.ts",
+        "test:storage-record-keys": "npx tsx --test tests/storage-record-keys.test.ts",
         "test:push-idempotency": "npx tsx --test tests/push-idempotency.test.ts",
         "test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
         "test:charging-client": "npx tsx --test tests/charging-client.test.ts",