scrapely-python 0.1.32__py3-none-any.whl → 0.1.34__py3-none-any.whl

scrapely/__init__.py

@@ -227,7 +227,7 @@ if TYPE_CHECKING:  # Static analysers and IDEs see the lazy names as plain impor
         set_api_storage,
     )
 
-__version__ = '0.1.32'
+__version__ = '0.1.34'
 __all__ = [
     # Core
     'Actor',

scrapely/actor.py

@@ -285,6 +285,53 @@ class _ActorMeta(type):
         await cls._shutdown()
 
 
+
+def _element_markup(obj: Any) -> Optional[str]:
+    """The markup of a parsed XML/HTML element, or None when ``obj`` is not one.
+
+    Debug captures store the source an item was read from. Feed and page parsers hand over elements
+    (xml.etree / lxml elements and trees, BeautifulSoup tags, selectolax nodes, parsel / Scrapy
+    selectors); their ``str()`` is only a repr such as ``<Element 'change' at 0x...>``, which shows
+    nothing of the source. Optional parsers are recognised by module, never imported here.
+    """
+    import xml.etree.ElementTree as ET
+
+    if isinstance(obj, ET.ElementTree):
+        root = obj.getroot()
+        return ET.tostring(root, encoding='unicode') if root is not None else ''
+    if isinstance(obj, ET.Element):
+        if obj.tail:  # the text after the element belongs to its parent, as with lxml's with_tail=False
+            import copy
+
+            obj = copy.copy(obj)
+            obj.tail = None
+        return ET.tostring(obj, encoding='unicode')
+    modules = [getattr(c, '__module__', '') or '' for c in type(obj).__mro__]
+    if any(m == 'lxml' or m.startswith('lxml.') for m in modules):
+        from lxml import etree  # type: ignore[import-not-found]
+
+        out = etree.tostring(obj, encoding='unicode', with_tail=False)
+        return out if isinstance(out, str) else None
+    if any(m == 'bs4' or m.startswith('bs4.') for m in modules):
+        return str(obj)
+    if any(m == 'selectolax' or m.startswith('selectolax.') for m in modules):
+        html = getattr(obj, 'html', None)
+        return html if isinstance(html, str) else None
+    if any(m == 'parsel' or m.startswith('parsel.') for m in modules) and callable(getattr(obj, 'get', None)):
+        out = obj.get()
+        return out if isinstance(out, str) else None
+    return None
+
+
+def _capture_json_default(obj: Any) -> Any:
+    """json.dumps fallback for debug captures: elements as their markup, anything else as str()."""
+    try:
+        markup = _element_markup(obj)
+    except Exception:  # an unusual node type must not lose the rest of the payload
+        markup = None
+    return markup if markup is not None else str(obj)
+
+
 class Actor(metaclass=_ActorMeta):
     """Core class for building Actors on the Scrapely platform."""
 
@@ -925,7 +972,14 @@ class Actor(metaclass=_ActorMeta):
 
     # Debug capture: raw source payloads for platform diagnostics. Only active when the platform
     # sets SCRAPELY_DEBUG_CAPTURE=1 on its own diagnostic runs; a no-op in every other run.
+    # A payload up to _DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+    # (base64, at most _DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to _DEBUG_CAPTURE_MAX_CHARS
+    # characters / _DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+    # needed when a large API document starts with site navigation (the seller section of a 581 KB
+    # listing JSON sat at ~333 KB).
     _DEBUG_CAPTURE_MAX_BYTES = 256 * 1024
+    _DEBUG_CAPTURE_MAX_CHARS = 8 * 1024 * 1024
+    _DEBUG_CAPTURE_MAX_ENCODED = 8 * 1024 * 1024
     _DEBUG_CAPTURE_MAX_RECORDS = 500
     _debug_capture_count = 0
 
@@ -967,15 +1021,36 @@ class Actor(metaclass=_ActorMeta):
         if cls._debug_capture_count >= cls._DEBUG_CAPTURE_MAX_RECORDS:
             return
         try:
-            text = payload if isinstance(payload, str) else json.dumps(payload, default=str, ensure_ascii=False)
-            if isinstance(text, bytes):
-                text = text.decode('utf-8', 'replace')
-            text = text[: cls._DEBUG_CAPTURE_MAX_BYTES]
+            markup = _element_markup(payload)
+            if markup is not None:
+                text = markup
+            elif isinstance(payload, (bytes, bytearray)):
+                text = bytes(payload).decode('utf-8', 'replace')
+            elif isinstance(payload, str):
+                text = payload
+            else:
+                text = json.dumps(payload, default=_capture_json_default, ensure_ascii=False, separators=(',', ':'))
+            record = cls._debug_capture_record(str(key), text)
             cls._debug_capture_count += 1
-            await cls.set_value(cls.debug_capture_record_key(key), {'key': str(key), 'payload': text})
+            await cls.set_value(cls.debug_capture_record_key(key), record)
         except Exception as err:  # never break the run
             logging.getLogger('scrapely').debug('debug_capture failed: %s', err)
 
+    @classmethod
+    def _debug_capture_record(cls, key: str, text: str) -> Dict[str, Any]:
+        """The record for one capture: plain when small, else gzip-base64 (plain head as the last resort)."""
+        limit = cls._DEBUG_CAPTURE_MAX_BYTES
+        if len(text) <= limit:
+            return {'key': key, 'payload': text}
+        import gzip
+        raw = text[: cls._DEBUG_CAPTURE_MAX_CHARS].encode('utf-8')
+        kept = raw[: cls._DEBUG_CAPTURE_MAX_ENCODED]
+        packed = base64.b64encode(gzip.compress(kept, 6)).decode('ascii')
+        if len(packed) <= limit:
+            truncated = len(text) > cls._DEBUG_CAPTURE_MAX_CHARS or len(kept) < len(raw)
+            return {'key': key, 'payload': packed, 'encoding': 'gzip-base64', 'length': len(text), 'truncated': truncated}
+        return {'key': key, 'payload': text[:limit], 'length': len(text), 'truncated': True}
+
     @classmethod
     def _resolve_open_args(
         cls,

{scrapely_python-0.1.32.dist-info → scrapely_python-0.1.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.32
+Version: 0.1.34
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.32.dist-info → scrapely_python-0.1.34.dist-info}/RECORD

@@ -1,6 +1,6 @@
-scrapely/__init__.py,sha256=10DwJc_P2SptqSlrJGlBxK_Tfp70RpDIf1Tuv0NVO5Q,9668
+scrapely/__init__.py,sha256=5lWyf7hHuDQXopBE8Koi2joSWNZWECE2DvPKu7CvkX0,9668
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
-scrapely/actor.py,sha256=mq5kchL3EIm_E1Y4uxj5UnzgUVJ7Gze33YXJNTOhUng,87611
+scrapely/actor.py,sha256=DmDnwPSmqmdEq_HIBgW0JivKl-8FrUBhNKNGXYh6QGA,91407
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
 scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
 scrapely/charging.py,sha256=osi2Ys99IyIJwFj0eRtV_RshokPs4mA2CE7oftNbN58,95559
@@ -55,10 +55,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=1C22qY-RWlARzMyW
 scrapely/storage/_file_system/_storage_client.py,sha256=QHa3IULEb6AmgRFwpBqHD2OUTmtA_gl3lId-j6QdSFo,2027
 scrapely/storage_clients/__init__.py,sha256=q2pC-Rwe-BfHl018OyN_P0EDSx4-hNdxbk3pOCLLd4U,618
 scrapely/storages/__init__.py,sha256=kDNtd9VodjdHd4u7GzruJo0aB2HeiBN2_-HrkeqjmFg,139
-scrapely_python-0.1.32.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
-scrapely_python-0.1.32.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
-scrapely_python-0.1.32.dist-info/licenses/NOTICE,sha256=1ltrzKk2ez7x2BPNpXeyvOlTty5AfABYDBlwNilpORk,788
-scrapely_python-0.1.32.dist-info/METADATA,sha256=YyH8wpWXWABQFwdL1I4o5HEB1OQbqiPGPPguoQ-a4mQ,3582
-scrapely_python-0.1.32.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-scrapely_python-0.1.32.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.32.dist-info/RECORD,,
+scrapely_python-0.1.34.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
+scrapely_python-0.1.34.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
+scrapely_python-0.1.34.dist-info/licenses/NOTICE,sha256=1ltrzKk2ez7x2BPNpXeyvOlTty5AfABYDBlwNilpORk,788
+scrapely_python-0.1.34.dist-info/METADATA,sha256=f84Jwc8hXM7jbHFipBYO1dlAojlHDxBpm1B8-BNmeYY,3582
+scrapely_python-0.1.34.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+scrapely_python-0.1.34.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.34.dist-info/RECORD,,