scrapely-python 0.1.32__tar.gz → 0.1.34__tar.gz

{scrapely_python-0.1.32 → scrapely_python-0.1.34}/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.1.34] - 2026-10-23
+
+- Debug captures (`Actor.debug_capture`, `Actor.debug_capture_sync`) store parsed XML/HTML elements as their markup instead of a repr such as `<Element 'change' at 0x...>`: `xml.etree.ElementTree` elements and trees, lxml elements and trees, BeautifulSoup tags, selectolax nodes and parsel / Scrapy selectors, passed alone or inside a dict or list. The text after an element (its tail) is not included. Other objects are still stored as their `str()`. Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).
+
+## [0.1.33] - 2026-10-22
+
+- Debug captures (`Actor.debug_capture`, `Actor.debug_capture_sync`) keep payloads over 256 KiB whole. They are stored gzip-compressed and base64-encoded as `{key, payload, encoding: 'gzip-base64', length, truncated}` instead of being cut at 262,144 characters; up to 8 Mi characters are kept. A payload that does not compress enough keeps its first 256 KiB, now marked `truncated: True` with its full `length`. Dict and list payloads are dumped as compact JSON, and `bytes` payloads are stored as UTF-8 text (they used to be stored as their `repr`). Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).
+
 ## [0.1.32] - 2026-10-20
 
 - Added `scrapely.scrapy`, the Scrapy integration: `run_scrapy_actor`, `initialize_logging`, `apply_apify_settings`, a scheduler backed by the run's request queue, a pipeline that pushes items to the default dataset, a proxy middleware, an HTTP cache storage and an extension that stops the crawl gracefully when the run is aborted. Queued Scrapy requests and cached responses are stored as JSON, never pickled, and a stored request only rebuilds as a `scrapy.Request` subclass. A request leaves the queue as handled only once Scrapy has finished it, so a migrated or aborted run does not lose or repeat it. Scrapy is imported only when these are used.

{scrapely_python-0.1.32 → scrapely_python-0.1.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.32
+Version: 0.1.34
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.32 → scrapely_python-0.1.34}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.32"
+version = "0.1.34"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.32 → scrapely_python-0.1.34}/src/scrapely/__init__.py

@@ -227,7 +227,7 @@ if TYPE_CHECKING:  # Static analysers and IDEs see the lazy names as plain impor
         set_api_storage,
     )
 
-__version__ = '0.1.32'
+__version__ = '0.1.34'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.32 → scrapely_python-0.1.34}/src/scrapely/actor.py

@@ -285,6 +285,53 @@ class _ActorMeta(type):
         await cls._shutdown()
 
 
+
+def _element_markup(obj: Any) -> Optional[str]:
+    """The markup of a parsed XML/HTML element, or None when ``obj`` is not one.
+
+    Debug captures store the source an item was read from. Feed and page parsers hand over elements
+    (xml.etree / lxml elements and trees, BeautifulSoup tags, selectolax nodes, parsel / Scrapy
+    selectors); their ``str()`` is only a repr such as ``<Element 'change' at 0x...>``, which shows
+    nothing of the source. Optional parsers are recognised by module, never imported here.
+    """
+    import xml.etree.ElementTree as ET
+
+    if isinstance(obj, ET.ElementTree):
+        root = obj.getroot()
+        return ET.tostring(root, encoding='unicode') if root is not None else ''
+    if isinstance(obj, ET.Element):
+        if obj.tail:  # the text after the element belongs to its parent, as with lxml's with_tail=False
+            import copy
+
+            obj = copy.copy(obj)
+            obj.tail = None
+        return ET.tostring(obj, encoding='unicode')
+    modules = [getattr(c, '__module__', '') or '' for c in type(obj).__mro__]
+    if any(m == 'lxml' or m.startswith('lxml.') for m in modules):
+        from lxml import etree  # type: ignore[import-not-found]
+
+        out = etree.tostring(obj, encoding='unicode', with_tail=False)
+        return out if isinstance(out, str) else None
+    if any(m == 'bs4' or m.startswith('bs4.') for m in modules):
+        return str(obj)
+    if any(m == 'selectolax' or m.startswith('selectolax.') for m in modules):
+        html = getattr(obj, 'html', None)
+        return html if isinstance(html, str) else None
+    if any(m == 'parsel' or m.startswith('parsel.') for m in modules) and callable(getattr(obj, 'get', None)):
+        out = obj.get()
+        return out if isinstance(out, str) else None
+    return None
+
+
+def _capture_json_default(obj: Any) -> Any:
+    """json.dumps fallback for debug captures: elements as their markup, anything else as str()."""
+    try:
+        markup = _element_markup(obj)
+    except Exception:  # an unusual node type must not lose the rest of the payload
+        markup = None
+    return markup if markup is not None else str(obj)
+
+
 class Actor(metaclass=_ActorMeta):
     """Core class for building Actors on the Scrapely platform."""
 
@@ -925,7 +972,14 @@ class Actor(metaclass=_ActorMeta):
 
     # Debug capture: raw source payloads for platform diagnostics. Only active when the platform
     # sets SCRAPELY_DEBUG_CAPTURE=1 on its own diagnostic runs; a no-op in every other run.
+    # A payload up to _DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+    # (base64, at most _DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to _DEBUG_CAPTURE_MAX_CHARS
+    # characters / _DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+    # needed when a large API document starts with site navigation (the seller section of a 581 KB
+    # listing JSON sat at ~333 KB).
     _DEBUG_CAPTURE_MAX_BYTES = 256 * 1024
+    _DEBUG_CAPTURE_MAX_CHARS = 8 * 1024 * 1024
+    _DEBUG_CAPTURE_MAX_ENCODED = 8 * 1024 * 1024
     _DEBUG_CAPTURE_MAX_RECORDS = 500
     _debug_capture_count = 0
 
@@ -967,15 +1021,36 @@ class Actor(metaclass=_ActorMeta):
         if cls._debug_capture_count >= cls._DEBUG_CAPTURE_MAX_RECORDS:
             return
         try:
-            text = payload if isinstance(payload, str) else json.dumps(payload, default=str, ensure_ascii=False)
-            if isinstance(text, bytes):
-                text = text.decode('utf-8', 'replace')
-            text = text[: cls._DEBUG_CAPTURE_MAX_BYTES]
+            markup = _element_markup(payload)
+            if markup is not None:
+                text = markup
+            elif isinstance(payload, (bytes, bytearray)):
+                text = bytes(payload).decode('utf-8', 'replace')
+            elif isinstance(payload, str):
+                text = payload
+            else:
+                text = json.dumps(payload, default=_capture_json_default, ensure_ascii=False, separators=(',', ':'))
+            record = cls._debug_capture_record(str(key), text)
             cls._debug_capture_count += 1
-            await cls.set_value(cls.debug_capture_record_key(key), {'key': str(key), 'payload': text})
+            await cls.set_value(cls.debug_capture_record_key(key), record)
         except Exception as err:  # never break the run
             logging.getLogger('scrapely').debug('debug_capture failed: %s', err)
 
+    @classmethod
+    def _debug_capture_record(cls, key: str, text: str) -> Dict[str, Any]:
+        """The record for one capture: plain when small, else gzip-base64 (plain head as the last resort)."""
+        limit = cls._DEBUG_CAPTURE_MAX_BYTES
+        if len(text) <= limit:
+            return {'key': key, 'payload': text}
+        import gzip
+        raw = text[: cls._DEBUG_CAPTURE_MAX_CHARS].encode('utf-8')
+        kept = raw[: cls._DEBUG_CAPTURE_MAX_ENCODED]
+        packed = base64.b64encode(gzip.compress(kept, 6)).decode('ascii')
+        if len(packed) <= limit:
+            truncated = len(text) > cls._DEBUG_CAPTURE_MAX_CHARS or len(kept) < len(raw)
+            return {'key': key, 'payload': packed, 'encoding': 'gzip-base64', 'length': len(text), 'truncated': truncated}
+        return {'key': key, 'payload': text[:limit], 'length': len(text), 'truncated': True}
+
     @classmethod
     def _resolve_open_args(
         cls,

{scrapely_python-0.1.32 → scrapely_python-0.1.34}/src/scrapely_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.32
+Version: 0.1.34
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0