scrapely-python 0.1.15__tar.gz → 0.1.17__tar.gz

{scrapely_python-0.1.15/src/scrapely_python.egg-info → scrapely_python-0.1.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.15
+Version: 0.1.17
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.15 → scrapely_python-0.1.17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.15"
+version = "0.1.17"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.15 → scrapely_python-0.1.17}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.15'
+__version__ = '0.1.17'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.15 → scrapely_python-0.1.17}/src/scrapely/actor.py

@@ -1080,6 +1080,34 @@ class Actor(metaclass=_ActorMeta):
             status_message=data.get('statusMessage'),
         )
 
+    @classmethod
+    async def solver_status(cls, vendor: str) -> Dict[str, Any]:
+        """Anti-bot solver status for a vendor (free, cached for 15 s, never raises).
+
+        ``usable`` False means the vendor cannot be solved right now: take your
+        fallback (for example the hosted browser), or fail the items that need
+        a solve at once and call ``set_solver_degraded_status()``. See
+        ``scrapely.solver``.
+        """
+        from .solver import solver_status_async
+
+        return await solver_status_async(vendor)
+
+    @classmethod
+    async def set_solver_degraded_status(cls, skipped: Optional[int] = None, total: Optional[int] = None) -> None:
+        """Set the standard status message for a run whose site protection cannot be solved right now.
+
+        "Site protection temporarily unavailable, results may be incomplete",
+        with "(N of M items skipped)" when both counts are given. Never raises:
+        the run goes on.
+        """
+        from .solver import degraded_status_message
+
+        try:
+            await cls.set_status_message(degraded_status_message(skipped, total))
+        except Exception as exc:  # noqa: BLE001 - a status message must never stop the run
+            logger.warning(f'Could not set the solver status message: {exc}')
+
     @classmethod
     async def _put_run_status_message(
         cls, message: str, *, is_terminal: bool

{scrapely_python-0.1.15 → scrapely_python-0.1.17}/src/scrapely/charging.py

@@ -270,6 +270,13 @@ class ChargingManager:
         self._charge_limit_hit = False
         # Default-dataset rows refused after the limit (one warning, then a count).
         self._refused_default_dataset_items = 0
+        # Prepaid-row balance: priced events charged through charge() minus rows
+        # pushed to the default dataset without a per-row charge. An Actor that
+        # charges first and then pushes plainly (charge('product'); push_data(row))
+        # has a positive balance after the charge, so the row it already paid for
+        # is still saved when that charge used up the spending limit. Push-then-
+        # charge Actors never go positive, so they get nothing extra for free.
+        self._prepaid_row_balance = 0
 
     def close(self) -> None:
         """Mark the manager closed; subsequent charge/push operations are rejected."""
@@ -323,8 +330,40 @@ class ChargingManager:
             )
         self._refused_default_dataset_items += count
 
+    def _note_prepaid_charge(self, event_name: str, charged_count: int) -> None:
+        if (
+            charged_count > 0
+            and not is_default_dataset_item_event(event_name)
+            and self._per_event_prices.get(event_name, 0) > 0
+        ):
+            self._prepaid_row_balance += charged_count
+
+    def _take_unpaid_rows(self, item_count: int) -> int:
+        """Account for ``item_count`` rows pushed without a per-row charge.
+
+        Returns how many of them may be stored: all of them while the limit is
+        not reached, afterwards only the rows already paid for by earlier
+        charges (the positive prepaid balance).
+        """
+        if item_count <= 0:
+            return 0
+        if not self.is_charge_limit_reached():
+            self._prepaid_row_balance -= item_count
+            return item_count
+        allowed = max(0, min(item_count, self._prepaid_row_balance))
+        self._prepaid_row_balance -= allowed
+        return allowed
+
     def _refuse_default_dataset_push(self, item_count: int) -> bool:
-        if item_count <= 0 or not self.is_charge_limit_reached():
+        if item_count <= 0 or self._pricing_info is None:
+            return False
+        if not self.is_charge_limit_reached():
+            self._take_unpaid_rows(item_count)
+            return False
+        # All-or-nothing here (storage clients cannot store a slice): store the
+        # batch only when earlier charges paid for every row of it.
+        if self._prepaid_row_balance >= item_count:
+            self._prepaid_row_balance -= item_count
             return False
         self._note_refused_default_dataset_items(item_count)
         return True
@@ -851,7 +890,9 @@ class ChargingManager:
     async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
         """Atomically reserve and submit one logical charge operation."""
         async with self._operation_lock:
-            return await self._charge_exclusive(event_name, count)
+            result = await self._charge_exclusive(event_name, count)
+            self._note_prepaid_charge(event_name, result.charged_count)
+            return result
 
     async def _charge_exclusive(self, event_name: str, count: int = 1) -> ChargeResult:
         if self._closed:
@@ -1327,19 +1368,27 @@ async def push_data_and_charge(
         # default-dataset push that does not pay per row (no event, or only
         # free events) stores nothing. Paid pushes go on below, where the
         # budget cap stores only the rows that are actually charged.
+        # Rows the Actor already paid for with an earlier charge() (charge first,
+        # then a plain push) are still stored: see ChargingManager._take_unpaid_rows.
+        # The result still reports the limit, so the Actor stops after them.
+        unpaid_past_limit = False
         if (
             is_ppe
             and is_default_dataset
             and items
-            and charging_manager.is_charge_limit_reached()
             and not _push_is_paid_per_item(charging_manager, event_name, resolved_event)
         ):
-            charging_manager._note_refused_default_dataset_items(len(items))
-            return ChargeResult(
-                chargeable_within_limit=charging_manager.compute_chargeable(),
-                charged_count=0,
-                event_charge_limit_reached=True,
-            )
+            unpaid_past_limit = charging_manager.is_charge_limit_reached()
+            allowed = charging_manager._take_unpaid_rows(len(items))
+            if allowed < len(items):
+                charging_manager._note_refused_default_dataset_items(len(items) - allowed)
+                if allowed == 0:
+                    return ChargeResult(
+                        chargeable_within_limit=charging_manager.compute_chargeable(),
+                        charged_count=0,
+                        event_charge_limit_reached=True,
+                    )
+                items = items[:allowed]
 
         limited_count = len(items)
         if is_ppe and resolved_event:
@@ -1522,7 +1571,8 @@ async def push_data_and_charge(
                     event_charge_limit_reached=any(
                         result.event_charge_limit_reached for result in results
                     )
-                    or accepted_for_items < len(limited_items),
+                    or accepted_for_items < len(limited_items)
+                    or unpaid_past_limit,
                 )
 
 
@@ -1531,7 +1581,7 @@ async def push_data_and_charge(
         return ChargeResult(
             chargeable_within_limit=charging_manager.compute_chargeable(),
             charged_count=0,
-            event_charge_limit_reached=False,
+            event_charge_limit_reached=unpaid_past_limit,
         )