scrapely-python 0.1.33__tar.gz → 0.1.35__tar.gz

{scrapely_python-0.1.33 → scrapely_python-0.1.35}/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.1.35] - 2026-10-26
+
+- Once `Actor.fail()` has run, the run stays failed. A later `Actor.exit()` keeps the failure's exit code and status message, so the usual `try: ... await Actor.fail(...) finally: await Actor.exit()` pattern now ends the run as FAILED. Before, the `exit()` in `finally` replaced the failure's exit code 1 with 0 and the run ended as SUCCEEDED with the failure message as its status. The same holds after `fail()` whose `SystemExit` was caught, after `fail(ExitOptions(exit=False))` (also inside `async with Actor(...)` and `Actor.main()`), and for a second `fail()`, which keeps the first failure's code and message.
+- `Actor.fail(ExitOptions(...))` without an exit code exits with 1. `ExitOptions` defaults `exit_code` to 0, which made such a failure exit with 0. A non-zero `exit_code` in the options, or an explicit `exit_code=` keyword, still wins.
+
+## [0.1.34] - 2026-10-23
+
+- Debug captures (`Actor.debug_capture`, `Actor.debug_capture_sync`) store parsed XML/HTML elements as their markup instead of a repr such as `<Element 'change' at 0x...>`: `xml.etree.ElementTree` elements and trees, lxml elements and trees, BeautifulSoup tags, selectolax nodes and parsel / Scrapy selectors, passed alone or inside a dict or list. The text after an element (its tail) is not included. Other objects are still stored as their `str()`. Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).
+
 ## [0.1.33] - 2026-10-22
 
 - Debug captures (`Actor.debug_capture`, `Actor.debug_capture_sync`) keep payloads over 256 KiB whole. They are stored gzip-compressed and base64-encoded as `{key, payload, encoding: 'gzip-base64', length, truncated}` instead of being cut at 262,144 characters; up to 8 Mi characters are kept. A payload that does not compress enough keeps its first 256 KiB, now marked `truncated: True` with its full `length`. Dict and list payloads are dumped as compact JSON, and `bytes` payloads are stored as UTF-8 text (they used to be stored as their `repr`). Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).

{scrapely_python-0.1.33 → scrapely_python-0.1.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.33
+Version: 0.1.35
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.33 → scrapely_python-0.1.35}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.33"
+version = "0.1.35"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.33 → scrapely_python-0.1.35}/src/scrapely/__init__.py

@@ -227,7 +227,7 @@ if TYPE_CHECKING:  # Static analysers and IDEs see the lazy names as plain impor
         set_api_storage,
     )
 
-__version__ = '0.1.33'
+__version__ = '0.1.35'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.33 → scrapely_python-0.1.35}/src/scrapely/actor.py

@@ -285,6 +285,53 @@ class _ActorMeta(type):
         await cls._shutdown()
 
 
+
+def _element_markup(obj: Any) -> Optional[str]:
+    """The markup of a parsed XML/HTML element, or None when ``obj`` is not one.
+
+    Debug captures store the source an item was read from. Feed and page parsers hand over elements
+    (xml.etree / lxml elements and trees, BeautifulSoup tags, selectolax nodes, parsel / Scrapy
+    selectors); their ``str()`` is only a repr such as ``<Element 'change' at 0x...>``, which shows
+    nothing of the source. Optional parsers are recognised by module, never imported here.
+    """
+    import xml.etree.ElementTree as ET
+
+    if isinstance(obj, ET.ElementTree):
+        root = obj.getroot()
+        return ET.tostring(root, encoding='unicode') if root is not None else ''
+    if isinstance(obj, ET.Element):
+        if obj.tail:  # the text after the element belongs to its parent, as with lxml's with_tail=False
+            import copy
+
+            obj = copy.copy(obj)
+            obj.tail = None
+        return ET.tostring(obj, encoding='unicode')
+    modules = [getattr(c, '__module__', '') or '' for c in type(obj).__mro__]
+    if any(m == 'lxml' or m.startswith('lxml.') for m in modules):
+        from lxml import etree  # type: ignore[import-not-found]
+
+        out = etree.tostring(obj, encoding='unicode', with_tail=False)
+        return out if isinstance(out, str) else None
+    if any(m == 'bs4' or m.startswith('bs4.') for m in modules):
+        return str(obj)
+    if any(m == 'selectolax' or m.startswith('selectolax.') for m in modules):
+        html = getattr(obj, 'html', None)
+        return html if isinstance(html, str) else None
+    if any(m == 'parsel' or m.startswith('parsel.') for m in modules) and callable(getattr(obj, 'get', None)):
+        out = obj.get()
+        return out if isinstance(out, str) else None
+    return None
+
+
+def _capture_json_default(obj: Any) -> Any:
+    """json.dumps fallback for debug captures: elements as their markup, anything else as str()."""
+    try:
+        markup = _element_markup(obj)
+    except Exception:  # an unusual node type must not lose the rest of the payload
+        markup = None
+    return markup if markup is not None else str(obj)
+
+
 class Actor(metaclass=_ActorMeta):
     """Core class for building Actors on the Scrapely platform."""
 
@@ -302,6 +349,10 @@ class Actor(metaclass=_ActorMeta):
     _cached_input: Optional[Dict[str, Any]] = None
     _exit_code: int = 0
     _status_message: Optional[str] = None
+    # Set by fail() for the rest of the lifecycle: (exit code, status message).
+    # A later exit() keeps them, so `fail()` followed by the usual `exit()` in a
+    # `finally:` block still ends the run as FAILED (it used to exit with 0).
+    _failure: Optional[Tuple[int, Optional[str]]] = None
     _charging_manager: Optional[ChargingManager] = None
     _use_state_stores: set = set()
     # Lifecycle bookkeeping set in init() / cleared in _shutdown().
@@ -345,9 +396,11 @@ class Actor(metaclass=_ActorMeta):
         elif exc_type is not None:
             cls._exit_code = 1
             logger.error(f'Actor failed: {exc_val}')
+        elif cls._failure is not None:
+            cls._exit_code = cls._failure[0]  # fail(exit=False) earlier: the run stays failed
         else:
             cls._exit_code = self._instance_exit_code
-        if self._instance_status_message:
+        if self._instance_status_message and cls._failure is None:
             cls._status_message = self._instance_status_message
             await cls._persist_terminal_status_message(self._instance_status_message)
         await cls._shutdown(timeout_secs=int(self._cleanup_timeout.total_seconds()))
@@ -363,6 +416,7 @@ class Actor(metaclass=_ActorMeta):
         cls._main_loop = asyncio.get_running_loop()  # for debug_capture_sync from worker threads
         if cls._initialized:
             return
+        cls._failure = None  # a new lifecycle starts unfailed
 
         if options is None or options.configure_logging:
             _lazy('_configure_logging')()
@@ -654,13 +708,33 @@ class Actor(metaclass=_ActorMeta):
         if exit_code is not None:
             opts.exit_code = exit_code
 
-        if opts.status_message:
+        if cls._failure is not None:
+            # Actor.fail() already ran: the run stays failed. Keep its exit code and
+            # status message; a later exit() (e.g. in a `finally:` block) only
+            # finishes the shutdown and exits with the failure's code.
+            if opts.status_message:
+                logger.info(f'[Scrapely] {opts.status_message}')
+            logger.info(
+                '[Scrapely] Actor.exit() after Actor.fail(): the run stays failed (exit code %s).',
+                cls._failure[0],
+            )
+            opts = ExitOptions(**{**opts.__dict__})
+            opts.exit_code, opts.status_message = cls._failure
+            await cls._exit_with(opts, persist_message=False)
+            return
+
+        await cls._exit_with(opts)
+
+    @classmethod
+    async def _exit_with(cls, opts: ExitOptions, persist_message: bool = True) -> None:
+        if opts.status_message and persist_message:
             logger.info(f'[Scrapely] {opts.status_message}')
 
         cls._exit_code = opts.exit_code
         cls._status_message = opts.status_message
 
-        await cls._persist_terminal_status_message(opts.status_message)
+        if persist_message:
+            await cls._persist_terminal_status_message(opts.status_message)
         await cls._shutdown(timeout_secs=opts.timeout_secs)
 
         if opts.exit:
@@ -689,7 +763,7 @@ class Actor(metaclass=_ActorMeta):
         if isinstance(message_or_options, str):
             opts = ExitOptions(exit_code=1, status_message=message_or_options)
         else:
-            # Options-object form: an explicit exit_code in the object wins over
+            # Options-object form: a non-zero exit_code in the object wins over
             # the fail() default of 1 (JS SDK semantics). Build the dict first —
             # passing it alongside exit_code=1 as kwargs raises
             # "got multiple values for keyword argument 'exit_code'".
@@ -704,12 +778,24 @@ class Actor(metaclass=_ActorMeta):
             opts.status_message = status_message
         if exit_code is not None:
             opts.exit_code = exit_code
+        elif not opts.exit_code:
+            # ExitOptions always carries exit_code (default 0), so 0 from an options
+            # object means "not set": a failed run must not exit with 0. Only an
+            # explicit exit_code=0 keyword keeps 0.
+            opts.exit_code = 1
 
         logger.error(
             f'[Scrapely] Actor failed: {opts.status_message or "Unknown error"}',
             exc_info=exception,
         )
-        await cls.exit(opts)
+        if cls._failure is not None:
+            # Failed already: the first failure's code and message stay.
+            opts = ExitOptions(**{**opts.__dict__})
+            opts.exit_code, opts.status_message = cls._failure
+            await cls._exit_with(opts, persist_message=False)
+            return
+        cls._failure = (opts.exit_code, opts.status_message)
+        await cls._exit_with(opts)
 
     @classmethod
     def is_at_home(cls) -> bool:
@@ -974,12 +1060,15 @@ class Actor(metaclass=_ActorMeta):
         if cls._debug_capture_count >= cls._DEBUG_CAPTURE_MAX_RECORDS:
             return
         try:
-            if isinstance(payload, (bytes, bytearray)):
+            markup = _element_markup(payload)
+            if markup is not None:
+                text = markup
+            elif isinstance(payload, (bytes, bytearray)):
                 text = bytes(payload).decode('utf-8', 'replace')
             elif isinstance(payload, str):
                 text = payload
             else:
-                text = json.dumps(payload, default=str, ensure_ascii=False, separators=(',', ':'))
+                text = json.dumps(payload, default=_capture_json_default, ensure_ascii=False, separators=(',', ':'))
             record = cls._debug_capture_record(str(key), text)
             cls._debug_capture_count += 1
             await cls.set_value(cls.debug_capture_record_key(key), record)

{scrapely_python-0.1.33 → scrapely_python-0.1.35}/src/scrapely_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.33
+Version: 0.1.35
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0