scrapely 1.2.35 → 1.2.36

package/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.2.36] - 2026-10-22
+
+- Debug captures (`Actor.debugCapture`, `Actor.debugCaptureSync`) keep payloads over 256 KiB whole. They are stored gzip-compressed and base64-encoded as `{ key, payload, encoding: 'gzip-base64', length, truncated }` instead of being cut at 262,144 characters; up to 8 Mi characters are kept. A payload that does not compress enough keeps its first 256 KiB, now marked `truncated: true` with its full `length`. `Uint8Array` / `Buffer` payloads are stored as UTF-8 text. Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).
+
 ## [1.2.35] - 2026-10-20
 
 - The storage client percent-encodes request ids and key-value store record keys as one URL path segment and refuses `''`, `.` and `..`. A request id such as `..` read from a queue used to send the call to another resource.

package/dist/actor.d.ts

@@ -459,7 +459,15 @@ export declare class Actor {
      */
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
     private static _debugCaptureCount;
+    /**
+     * A payload up to DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+     * (base64, at most DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to DEBUG_CAPTURE_MAX_CHARS
+     * characters / DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+     * needed when a large API document starts with site navigation.
+     */
     static readonly DEBUG_CAPTURE_MAX_BYTES: number;
+    static readonly DEBUG_CAPTURE_MAX_CHARS: number;
+    static readonly DEBUG_CAPTURE_MAX_ENCODED: number;
     static readonly DEBUG_CAPTURE_MAX_RECORDS = 500;
     /** The key-value store record name for a capture key (item URL or id). */
     static debugCaptureRecordKey(key: string): Promise<string>;
@@ -471,6 +479,8 @@ export declare class Actor {
     /** debugCapture for synchronous code: fire and forget. Does nothing unless capture is enabled for the run. */
     static debugCaptureSync(key: string, payload: unknown): void;
     static debugCapture(key: string, payload: unknown): Promise<void>;
+    /** The record for one capture: plain when small, else gzip-base64 (plain head as the last resort). */
+    static debugCaptureRecord(key: string, text: string): Promise<Record<string, unknown>>;
     /**
      * Helper to resolve storage identifier to a string ID or name.
      * Supports: string, { id: '...' }, { name: '...' }, { alias: '...' }

package/dist/actor.js

@@ -660,7 +660,15 @@ class Actor {
         await store.setValue(key, value, options);
     }
     static _debugCaptureCount = 0;
+    /**
+     * A payload up to DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+     * (base64, at most DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to DEBUG_CAPTURE_MAX_CHARS
+     * characters / DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+     * needed when a large API document starts with site navigation.
+     */
     static DEBUG_CAPTURE_MAX_BYTES = 256 * 1024;
+    static DEBUG_CAPTURE_MAX_CHARS = 8 * 1024 * 1024;
+    static DEBUG_CAPTURE_MAX_ENCODED = 8 * 1024 * 1024;
     static DEBUG_CAPTURE_MAX_RECORDS = 500;
     /** The key-value store record name for a capture key (item URL or id). */
     static async debugCaptureRecordKey(key) {
@@ -684,14 +692,32 @@ class Actor {
         if (Actor._debugCaptureCount >= Actor.DEBUG_CAPTURE_MAX_RECORDS)
             return;
         try {
-            const text = (typeof payload === 'string' ? payload : JSON.stringify(payload) ?? '').slice(0, Actor.DEBUG_CAPTURE_MAX_BYTES);
+            const text = typeof payload === 'string' ? payload
+                : payload instanceof Uint8Array ? Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength).toString('utf8')
+                    : JSON.stringify(payload) ?? '';
+            const record = await Actor.debugCaptureRecord(String(key), text);
             Actor._debugCaptureCount++;
-            await Actor.setValue(await Actor.debugCaptureRecordKey(key), { key: String(key), payload: text });
+            await Actor.setValue(await Actor.debugCaptureRecordKey(key), record);
         }
         catch {
             // never break the run
         }
     }
+    /** The record for one capture: plain when small, else gzip-base64 (plain head as the last resort). */
+    static async debugCaptureRecord(key, text) {
+        const limit = Actor.DEBUG_CAPTURE_MAX_BYTES;
+        if (text.length <= limit)
+            return { key, payload: text };
+        const { gzipSync } = await import('node:zlib');
+        const raw = Buffer.from(text.slice(0, Actor.DEBUG_CAPTURE_MAX_CHARS), 'utf8');
+        const kept = raw.subarray(0, Actor.DEBUG_CAPTURE_MAX_ENCODED);
+        const packed = gzipSync(kept, { level: 6 }).toString('base64');
+        if (packed.length <= limit) {
+            const truncated = text.length > Actor.DEBUG_CAPTURE_MAX_CHARS || kept.length < raw.length;
+            return { key, payload: packed, encoding: 'gzip-base64', length: text.length, truncated };
+        }
+        return { key, payload: text.slice(0, limit), length: text.length, truncated: true };
+    }
     /**
      * Helper to resolve storage identifier to a string ID or name.
      * Supports: string, { id: '...' }, { name: '...' }, { alias: '...' }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.35",
+    "version": "1.2.36",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -48,6 +48,7 @@
         "test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
         "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
         "test:storage-path-segments": "npx tsx --test tests/storage-path-segments.test.ts",
+        "test:storage-record-keys": "npx tsx --test tests/storage-record-keys.test.ts",
         "test:push-idempotency": "npx tsx --test tests/push-idempotency.test.ts",
         "test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
         "test:charging-client": "npx tsx --test tests/charging-client.test.ts",