scrapely-python 0.1.32__tar.gz → 0.1.33__tar.gz

{scrapely_python-0.1.32 → scrapely_python-0.1.33}/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.1.33] - 2026-10-22
+
+- Debug captures (`Actor.debug_capture`, `Actor.debug_capture_sync`) keep payloads over 256 KiB whole. They are stored gzip-compressed and base64-encoded as `{key, payload, encoding: 'gzip-base64', length, truncated}` instead of being cut at 262,144 characters; up to 8 Mi characters are kept. A payload that does not compress enough keeps its first 256 KiB, now marked `truncated: True` with its full `length`. Dict and list payloads are dumped as compact JSON, and `bytes` payloads are stored as UTF-8 text (they used to be stored as their `repr`). Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).
+
 ## [0.1.32] - 2026-10-20
 
 - Added `scrapely.scrapy`, the Scrapy integration: `run_scrapy_actor`, `initialize_logging`, `apply_apify_settings`, a scheduler backed by the run's request queue, a pipeline that pushes items to the default dataset, a proxy middleware, an HTTP cache storage and an extension that stops the crawl gracefully when the run is aborted. Queued Scrapy requests and cached responses are stored as JSON, never pickled, and a stored request only rebuilds as a `scrapy.Request` subclass. A request leaves the queue as handled only once Scrapy has finished it, so a migrated or aborted run does not lose or repeat it. Scrapy is imported only when these are used.

{scrapely_python-0.1.32 → scrapely_python-0.1.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.32
+Version: 0.1.33
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.32 → scrapely_python-0.1.33}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.32"
+version = "0.1.33"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.32 → scrapely_python-0.1.33}/src/scrapely/__init__.py

@@ -227,7 +227,7 @@ if TYPE_CHECKING:  # Static analysers and IDEs see the lazy names as plain impor
         set_api_storage,
     )
 
-__version__ = '0.1.32'
+__version__ = '0.1.33'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.32 → scrapely_python-0.1.33}/src/scrapely/actor.py

@@ -925,7 +925,14 @@ class Actor(metaclass=_ActorMeta):
 
     # Debug capture: raw source payloads for platform diagnostics. Only active when the platform
     # sets SCRAPELY_DEBUG_CAPTURE=1 on its own diagnostic runs; a no-op in every other run.
+    # A payload up to _DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+    # (base64, at most _DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to _DEBUG_CAPTURE_MAX_CHARS
+    # characters / _DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+    # needed when a large API document starts with site navigation (the seller section of a 581 KB
+    # listing JSON sat at ~333 KB).
     _DEBUG_CAPTURE_MAX_BYTES = 256 * 1024
+    _DEBUG_CAPTURE_MAX_CHARS = 8 * 1024 * 1024
+    _DEBUG_CAPTURE_MAX_ENCODED = 8 * 1024 * 1024
     _DEBUG_CAPTURE_MAX_RECORDS = 500
     _debug_capture_count = 0
 
@@ -967,15 +974,33 @@ class Actor(metaclass=_ActorMeta):
         if cls._debug_capture_count >= cls._DEBUG_CAPTURE_MAX_RECORDS:
             return
         try:
-            text = payload if isinstance(payload, str) else json.dumps(payload, default=str, ensure_ascii=False)
-            if isinstance(text, bytes):
-                text = text.decode('utf-8', 'replace')
-            text = text[: cls._DEBUG_CAPTURE_MAX_BYTES]
+            if isinstance(payload, (bytes, bytearray)):
+                text = bytes(payload).decode('utf-8', 'replace')
+            elif isinstance(payload, str):
+                text = payload
+            else:
+                text = json.dumps(payload, default=str, ensure_ascii=False, separators=(',', ':'))
+            record = cls._debug_capture_record(str(key), text)
             cls._debug_capture_count += 1
-            await cls.set_value(cls.debug_capture_record_key(key), {'key': str(key), 'payload': text})
+            await cls.set_value(cls.debug_capture_record_key(key), record)
         except Exception as err:  # never break the run
             logging.getLogger('scrapely').debug('debug_capture failed: %s', err)
 
+    @classmethod
+    def _debug_capture_record(cls, key: str, text: str) -> Dict[str, Any]:
+        """The record for one capture: plain when small, else gzip-base64 (plain head as the last resort)."""
+        limit = cls._DEBUG_CAPTURE_MAX_BYTES
+        if len(text) <= limit:
+            return {'key': key, 'payload': text}
+        import gzip
+        raw = text[: cls._DEBUG_CAPTURE_MAX_CHARS].encode('utf-8')
+        kept = raw[: cls._DEBUG_CAPTURE_MAX_ENCODED]
+        packed = base64.b64encode(gzip.compress(kept, 6)).decode('ascii')
+        if len(packed) <= limit:
+            truncated = len(text) > cls._DEBUG_CAPTURE_MAX_CHARS or len(kept) < len(raw)
+            return {'key': key, 'payload': packed, 'encoding': 'gzip-base64', 'length': len(text), 'truncated': truncated}
+        return {'key': key, 'payload': text[:limit], 'length': len(text), 'truncated': True}
+
     @classmethod
     def _resolve_open_args(
         cls,

{scrapely_python-0.1.32 → scrapely_python-0.1.33}/src/scrapely_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.32
+Version: 0.1.33
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0