scrapely-python 0.1.16__tar.gz → 0.1.17__tar.gz

{scrapely_python-0.1.16/src/scrapely_python.egg-info → scrapely_python-0.1.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.16
+Version: 0.1.17
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.16 → scrapely_python-0.1.17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.16"
+version = "0.1.17"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.16 → scrapely_python-0.1.17}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.16'
+__version__ = '0.1.17'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.16 → scrapely_python-0.1.17}/src/scrapely/charging.py

@@ -270,6 +270,13 @@ class ChargingManager:
         self._charge_limit_hit = False
         # Default-dataset rows refused after the limit (one warning, then a count).
         self._refused_default_dataset_items = 0
+        # Prepaid-row balance: priced events charged through charge() minus rows
+        # pushed to the default dataset without a per-row charge. An Actor that
+        # charges first and then pushes plainly (charge('product'); push_data(row))
+        # has a positive balance after the charge, so the row it already paid for
+        # is still saved when that charge used up the spending limit. Push-then-
+        # charge Actors never go positive, so they get nothing extra for free.
+        self._prepaid_row_balance = 0
 
     def close(self) -> None:
         """Mark the manager closed; subsequent charge/push operations are rejected."""
@@ -323,8 +330,40 @@ class ChargingManager:
             )
         self._refused_default_dataset_items += count
 
+    def _note_prepaid_charge(self, event_name: str, charged_count: int) -> None:
+        if (
+            charged_count > 0
+            and not is_default_dataset_item_event(event_name)
+            and self._per_event_prices.get(event_name, 0) > 0
+        ):
+            self._prepaid_row_balance += charged_count
+
+    def _take_unpaid_rows(self, item_count: int) -> int:
+        """Account for ``item_count`` rows pushed without a per-row charge.
+
+        Returns how many of them may be stored: all of them while the limit is
+        not reached, afterwards only the rows already paid for by earlier
+        charges (the positive prepaid balance).
+        """
+        if item_count <= 0:
+            return 0
+        if not self.is_charge_limit_reached():
+            self._prepaid_row_balance -= item_count
+            return item_count
+        allowed = max(0, min(item_count, self._prepaid_row_balance))
+        self._prepaid_row_balance -= allowed
+        return allowed
+
     def _refuse_default_dataset_push(self, item_count: int) -> bool:
-        if item_count <= 0 or not self.is_charge_limit_reached():
+        if item_count <= 0 or self._pricing_info is None:
+            return False
+        if not self.is_charge_limit_reached():
+            self._take_unpaid_rows(item_count)
+            return False
+        # All-or-nothing here (storage clients cannot store a slice): store the
+        # batch only when earlier charges paid for every row of it.
+        if self._prepaid_row_balance >= item_count:
+            self._prepaid_row_balance -= item_count
             return False
         self._note_refused_default_dataset_items(item_count)
         return True
@@ -851,7 +890,9 @@ class ChargingManager:
     async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
         """Atomically reserve and submit one logical charge operation."""
         async with self._operation_lock:
-            return await self._charge_exclusive(event_name, count)
+            result = await self._charge_exclusive(event_name, count)
+            self._note_prepaid_charge(event_name, result.charged_count)
+            return result
 
     async def _charge_exclusive(self, event_name: str, count: int = 1) -> ChargeResult:
         if self._closed:
@@ -1327,19 +1368,27 @@ async def push_data_and_charge(
         # default-dataset push that does not pay per row (no event, or only
         # free events) stores nothing. Paid pushes go on below, where the
         # budget cap stores only the rows that are actually charged.
+        # Rows the Actor already paid for with an earlier charge() (charge first,
+        # then a plain push) are still stored: see ChargingManager._take_unpaid_rows.
+        # The result still reports the limit, so the Actor stops after them.
+        unpaid_past_limit = False
         if (
             is_ppe
             and is_default_dataset
             and items
-            and charging_manager.is_charge_limit_reached()
             and not _push_is_paid_per_item(charging_manager, event_name, resolved_event)
         ):
-            charging_manager._note_refused_default_dataset_items(len(items))
-            return ChargeResult(
-                chargeable_within_limit=charging_manager.compute_chargeable(),
-                charged_count=0,
-                event_charge_limit_reached=True,
-            )
+            unpaid_past_limit = charging_manager.is_charge_limit_reached()
+            allowed = charging_manager._take_unpaid_rows(len(items))
+            if allowed < len(items):
+                charging_manager._note_refused_default_dataset_items(len(items) - allowed)
+                if allowed == 0:
+                    return ChargeResult(
+                        chargeable_within_limit=charging_manager.compute_chargeable(),
+                        charged_count=0,
+                        event_charge_limit_reached=True,
+                    )
+                items = items[:allowed]
 
         limited_count = len(items)
         if is_ppe and resolved_event:
@@ -1522,7 +1571,8 @@ async def push_data_and_charge(
                     event_charge_limit_reached=any(
                         result.event_charge_limit_reached for result in results
                     )
-                    or accepted_for_items < len(limited_items),
+                    or accepted_for_items < len(limited_items)
+                    or unpaid_past_limit,
                 )
 
 
@@ -1531,7 +1581,7 @@ async def push_data_and_charge(
         return ChargeResult(
             chargeable_within_limit=charging_manager.compute_chargeable(),
             charged_count=0,
-            event_charge_limit_reached=False,
+            event_charge_limit_reached=unpaid_past_limit,
         )
 
 

{scrapely_python-0.1.16 → scrapely_python-0.1.17/src/scrapely_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.16
+Version: 0.1.17
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.16 → scrapely_python-0.1.17}/tests/test_charge_limit_guard.py

@@ -289,3 +289,114 @@ async def test_apify_client_hooks_intercept_default_dataset_past_limit() -> None
     assert hooks.should_charge_dataset_pushes(DEFAULT_DATASET_ID) is False
     await mgr.charge('result', 1)
     assert hooks.should_charge_dataset_pushes(DEFAULT_DATASET_ID) is True
+
+
+# ---- charge first, then a plain push (the row was paid before it was pushed) ----
+
+
+@pytest.mark.asyncio
+async def test_charge_then_plain_push_keeps_the_row_whose_charge_hit_the_cap() -> None:
+    """charge('result'); push_data(row): the last paid row must be saved, nothing after it."""
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
+    mgr = _manager(api)
+    store = Store()
+
+    for n in range(10):
+        r = await mgr.charge('result', 1)
+        if r.charged_count:
+            await _plain_push(mgr, store, [{'n': n}])
+        else:
+            await _plain_push(mgr, store, [{'n': n, 'unpaid': True}])
+
+    assert [i['n'] for i in store.items] == [0, 1, 2]
+    assert api.spent == Decimal('0.03')
+    assert mgr.is_charge_limit_reached() is True
+
+
+@pytest.mark.asyncio
+async def test_prepaid_rows_cap_a_bigger_plain_push() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.02'))
+    mgr = _manager(api, max_usd='0.02')
+    store = Store()
+
+    await mgr.charge('result', 2)
+    assert mgr.is_charge_limit_reached() is True
+    r = await _plain_push(mgr, store, [{'n': 1}, {'n': 2}, {'n': 3}])
+    assert [i['n'] for i in store.items] == [1, 2]
+    assert r.event_charge_limit_reached is True
+    assert mgr._refused_default_dataset_items == 1
+    # Balance used up: the next plain row is refused.
+    await _plain_push(mgr, store, [{'n': 4}])
+    assert len(store.items) == 2
+
+
+@pytest.mark.asyncio
+async def test_push_then_charge_actor_gets_no_free_row_after_the_cap() -> None:
+    """Balance never goes positive for push-then-charge, so the cap still stops it."""
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
+    mgr = _manager(api)
+    store = Store()
+
+    for n in range(6):
+        await _plain_push(mgr, store, [{'n': n}])
+        await mgr.charge('result', 1)
+
+    assert [i['n'] for i in store.items] == [0, 1, 2]
+    assert mgr._prepaid_row_balance <= 0
+
+
+@pytest.mark.asyncio
+async def test_unpriced_or_free_charges_do_not_prepay_rows() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
+    mgr = _manager(api, prices={'result': Decimal('0.01'), 'free-event': Decimal('0')}, max_usd='0.01')
+    store = Store()
+
+    await mgr.charge('result', 1)
+    await _plain_push(mgr, store, [{'n': 1}])  # paid by the charge above
+    await mgr.charge('free-event', 3)
+    await mgr.charge('not-in-pricing', 3)
+    await _plain_push(mgr, store, [{'n': 2}])
+    assert [i['n'] for i in store.items] == [1]
+
+
+@pytest.mark.asyncio
+async def test_storage_client_push_after_prepaying_charge_is_stored() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
+    mgr = _manager(api, max_usd='0.01')
+    set_active_charging_manager(mgr)
+    client, pushed = _dataset_client(DEFAULT_DATASET_ID)
+
+    await mgr.charge('result', 1)  # uses the whole budget, pays for one row
+    await client.push_data([{'n': 1}, {'n': 2}])  # two rows, one paid: whole batch refused
+    await client.push_data([{'n': 3}])  # one paid row: stored
+    await client.push_data([{'n': 4}])  # nothing left: refused
+
+    assert pushed == [(DEFAULT_DATASET_ID, [{'n': 3}])]
+
+
+@pytest.mark.asyncio
+async def test_partial_acceptance_saves_only_the_accepted_rows() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
+    mgr = _manager(api, max_usd='5')
+    store = Store()
+
+    c = await mgr.charge('result', 5)
+    assert c.charged_count == 3
+    r = await _plain_push(mgr, store, [{'n': n} for n in range(1, 6)])
+    assert [i['n'] for i in store.items] == [1, 2, 3]
+    assert r.event_charge_limit_reached is True
+    await _plain_push(mgr, store, [{'n': 6}])
+    assert len(store.items) == 3
+
+
+@pytest.mark.asyncio
+async def test_refused_charge_402_then_plain_push_is_not_saved() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=None, reject_after=2)
+    mgr = _manager(api, max_usd='5')
+    store = Store()
+
+    for n in range(4):
+        await mgr.charge('result', 1)
+        await _plain_push(mgr, store, [{'n': n}])
+
+    assert [i['n'] for i in store.items] == [0, 1]