scrapely-python 0.1.14__tar.gz → 0.1.16__tar.gz

{scrapely_python-0.1.14/src/scrapely_python.egg-info → scrapely_python-0.1.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.14
+Version: 0.1.16
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.14 → scrapely_python-0.1.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.14"
+version = "0.1.16"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.14 → scrapely_python-0.1.16}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.14'
+__version__ = '0.1.16'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.14 → scrapely_python-0.1.16}/src/scrapely/actor.py

@@ -1054,7 +1054,19 @@ class Actor(metaclass=_ActorMeta):
         else:
             logger.info(logged)
 
-        result = await cls._put_run_status_message(message, is_terminal=is_terminal)
+        if is_terminal:
+            result = await cls._put_run_status_message(message, is_terminal=True)
+        else:
+            # A progress update is best effort: if the API still fails after the
+            # retry window (e.g. a long 503), log it and carry on instead of
+            # crashing the run. Terminal updates keep raising.
+            try:
+                result = await cls._put_run_status_message(message, is_terminal=False)
+            except asyncio.CancelledError:
+                raise
+            except Exception as e:  # noqa: BLE001
+                logger.warning('Failed to update the status message (continuing): %s', e)
+                return ActorRun(id='', actor_id='', status='')
         if result is None:
             return ActorRun(id='', actor_id='', status='')
         run_id, data = result
@@ -1068,6 +1080,34 @@ class Actor(metaclass=_ActorMeta):
             status_message=data.get('statusMessage'),
         )
 
+    @classmethod
+    async def solver_status(cls, vendor: str) -> Dict[str, Any]:
+        """Anti-bot solver status for a vendor (free, cached for 15 s, never raises).
+
+        ``usable`` False means the vendor cannot be solved right now: take your
+        fallback (for example the hosted browser), or fail the items that need
+        a solve at once and call ``set_solver_degraded_status()``. See
+        ``scrapely.solver``.
+        """
+        from .solver import solver_status_async
+
+        return await solver_status_async(vendor)
+
+    @classmethod
+    async def set_solver_degraded_status(cls, skipped: Optional[int] = None, total: Optional[int] = None) -> None:
+        """Set the standard status message for a run whose site protection cannot be solved right now.
+
+        "Site protection temporarily unavailable, results may be incomplete",
+        with "(N of M items skipped)" when both counts are given. Never raises:
+        the run goes on.
+        """
+        from .solver import degraded_status_message
+
+        try:
+            await cls.set_status_message(degraded_status_message(skipped, total))
+        except Exception as exc:  # noqa: BLE001 - a status message must never stop the run
+            logger.warning(f'Could not set the solver status message: {exc}')
+
     @classmethod
     async def _put_run_status_message(
         cls, message: str, *, is_terminal: bool

{scrapely_python-0.1.14 → scrapely_python-0.1.16}/src/scrapely/charging.py

@@ -52,7 +52,7 @@ _PENDING_MAX_ENTRIES = 1024
 # Same-key POST attempts before leaving an intent pending / re-queueing a void.
 # The API client itself retries 429 and 5xx answers and refused or reset
 # connections of these POSTs with exponential backoff (honouring Retry-After)
-# for its retry window (2 minutes, SCRAPELY_API_RETRY_WINDOW_SECS): re-sending
+# for its retry window (5 minutes, SCRAPELY_API_RETRY_WINDOW_SECS): re-sending
 # is safe because every attempt carries the same Idempotency-Key. These outer
 # attempts cover what the client gives up on quickly (timeouts, other
 # transport errors), within _RETRY_BUDGET_SECS. Then the charge stays pending
@@ -62,7 +62,7 @@ _POST_ATTEMPTS = 8
 
 # Outer retries stop once this much time has passed since the first attempt
 # (the API retry window; JS SDK parity).
-_RETRY_BUDGET_SECS = 120.0
+_RETRY_BUDGET_SECS = 300.0
 
 
 def _is_transient_status(status: Optional[int]) -> bool:
@@ -1567,10 +1567,20 @@ def _push_certainly_failed(err: BaseException) -> bool:
     """True when a dataset push certainly stored nothing.
 
     The storage API answered with an error status, or the connection was never
-    made. Timeouts and resets after sending are not proof: the write may have landed.
+    made. Timeouts and resets after sending are not proof: the write may have landed,
+    and neither is any error of a push one of whose earlier attempts may have been
+    applied (``outcome_unknown``, set by the storage client's retries).
     """
     import httpx
 
+    c: Optional[BaseException] = err
+    for _ in range(4):
+        if c is None:
+            break
+        if getattr(c, 'outcome_unknown', False) is True:
+            return False
+        c = c.__cause__ or c.__context__
+
     e: Optional[BaseException] = err
     for _ in range(4):
         if e is None: