scrapely-python 0.1.32__py3-none-any.whl → 0.1.33__py3-none-any.whl

scrapely/__init__.py

@@ -227,7 +227,7 @@ if TYPE_CHECKING:  # Static analysers and IDEs see the lazy names as plain impor
         set_api_storage,
     )
 
-__version__ = '0.1.32'
+__version__ = '0.1.33'
 __all__ = [
     # Core
     'Actor',

scrapely/actor.py

@@ -925,7 +925,14 @@ class Actor(metaclass=_ActorMeta):
 
     # Debug capture: raw source payloads for platform diagnostics. Only active when the platform
     # sets SCRAPELY_DEBUG_CAPTURE=1 on its own diagnostic runs; a no-op in every other run.
+    # A payload up to _DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+    # (base64, at most _DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to _DEBUG_CAPTURE_MAX_CHARS
+    # characters / _DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+    # needed when a large API document starts with site navigation (the seller section of a 581 KB
+    # listing JSON sat at ~333 KB).
     _DEBUG_CAPTURE_MAX_BYTES = 256 * 1024
+    _DEBUG_CAPTURE_MAX_CHARS = 8 * 1024 * 1024
+    _DEBUG_CAPTURE_MAX_ENCODED = 8 * 1024 * 1024
     _DEBUG_CAPTURE_MAX_RECORDS = 500
     _debug_capture_count = 0
 
@@ -967,15 +974,33 @@ class Actor(metaclass=_ActorMeta):
         if cls._debug_capture_count >= cls._DEBUG_CAPTURE_MAX_RECORDS:
             return
         try:
-            text = payload if isinstance(payload, str) else json.dumps(payload, default=str, ensure_ascii=False)
-            if isinstance(text, bytes):
-                text = text.decode('utf-8', 'replace')
-            text = text[: cls._DEBUG_CAPTURE_MAX_BYTES]
+            if isinstance(payload, (bytes, bytearray)):
+                text = bytes(payload).decode('utf-8', 'replace')
+            elif isinstance(payload, str):
+                text = payload
+            else:
+                text = json.dumps(payload, default=str, ensure_ascii=False, separators=(',', ':'))
+            record = cls._debug_capture_record(str(key), text)
             cls._debug_capture_count += 1
-            await cls.set_value(cls.debug_capture_record_key(key), {'key': str(key), 'payload': text})
+            await cls.set_value(cls.debug_capture_record_key(key), record)
         except Exception as err:  # never break the run
             logging.getLogger('scrapely').debug('debug_capture failed: %s', err)
 
+    @classmethod
+    def _debug_capture_record(cls, key: str, text: str) -> Dict[str, Any]:
+        """The record for one capture: plain when small, else gzip-base64 (plain head as the last resort)."""
+        limit = cls._DEBUG_CAPTURE_MAX_BYTES
+        if len(text) <= limit:
+            return {'key': key, 'payload': text}
+        import gzip
+        raw = text[: cls._DEBUG_CAPTURE_MAX_CHARS].encode('utf-8')
+        kept = raw[: cls._DEBUG_CAPTURE_MAX_ENCODED]
+        packed = base64.b64encode(gzip.compress(kept, 6)).decode('ascii')
+        if len(packed) <= limit:
+            truncated = len(text) > cls._DEBUG_CAPTURE_MAX_CHARS or len(kept) < len(raw)
+            return {'key': key, 'payload': packed, 'encoding': 'gzip-base64', 'length': len(text), 'truncated': truncated}
+        return {'key': key, 'payload': text[:limit], 'length': len(text), 'truncated': True}
+
     @classmethod
     def _resolve_open_args(
         cls,

{scrapely_python-0.1.32.dist-info → scrapely_python-0.1.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.32
+Version: 0.1.33
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.32.dist-info → scrapely_python-0.1.33.dist-info}/RECORD

@@ -1,6 +1,6 @@
-scrapely/__init__.py,sha256=10DwJc_P2SptqSlrJGlBxK_Tfp70RpDIf1Tuv0NVO5Q,9668
+scrapely/__init__.py,sha256=IewlIO5DRdcqs_defs-pLoIJ_u78Vb7J7Ix0uMhV-ds,9668
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
-scrapely/actor.py,sha256=mq5kchL3EIm_E1Y4uxj5UnzgUVJ7Gze33YXJNTOhUng,87611
+scrapely/actor.py,sha256=mnQ0zYCsRJEAipxH1Sdg6o6_r2gEAvnuA1r5M3AsEgo,89122
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
 scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
 scrapely/charging.py,sha256=osi2Ys99IyIJwFj0eRtV_RshokPs4mA2CE7oftNbN58,95559
@@ -55,10 +55,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=1C22qY-RWlARzMyW
 scrapely/storage/_file_system/_storage_client.py,sha256=QHa3IULEb6AmgRFwpBqHD2OUTmtA_gl3lId-j6QdSFo,2027
 scrapely/storage_clients/__init__.py,sha256=q2pC-Rwe-BfHl018OyN_P0EDSx4-hNdxbk3pOCLLd4U,618
 scrapely/storages/__init__.py,sha256=kDNtd9VodjdHd4u7GzruJo0aB2HeiBN2_-HrkeqjmFg,139
-scrapely_python-0.1.32.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
-scrapely_python-0.1.32.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
-scrapely_python-0.1.32.dist-info/licenses/NOTICE,sha256=1ltrzKk2ez7x2BPNpXeyvOlTty5AfABYDBlwNilpORk,788
-scrapely_python-0.1.32.dist-info/METADATA,sha256=YyH8wpWXWABQFwdL1I4o5HEB1OQbqiPGPPguoQ-a4mQ,3582
-scrapely_python-0.1.32.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-scrapely_python-0.1.32.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.32.dist-info/RECORD,,
+scrapely_python-0.1.33.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
+scrapely_python-0.1.33.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
+scrapely_python-0.1.33.dist-info/licenses/NOTICE,sha256=1ltrzKk2ez7x2BPNpXeyvOlTty5AfABYDBlwNilpORk,788
+scrapely_python-0.1.33.dist-info/METADATA,sha256=hhDlcnp5JyIcT9C_-kawPgmqmGanpQgX3TJJqcCQl7s,3582
+scrapely_python-0.1.33.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+scrapely_python-0.1.33.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.33.dist-info/RECORD,,