scrapely-python 0.1.19__tar.gz → 0.1.21__tar.gz

{scrapely_python-0.1.19/src/scrapely_python.egg-info → scrapely_python-0.1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.19
+Version: 0.1.21
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.19 → scrapely_python-0.1.21}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.19"
+version = "0.1.21"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.19 → scrapely_python-0.1.21}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.19'
+__version__ = '0.1.21'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.19 → scrapely_python-0.1.21}/src/scrapely/actor.py

@@ -783,6 +783,7 @@ class Actor(metaclass=_ActorMeta):
         *,
         event_name: Optional[str] = None,
         charges: Optional[Dict[str, int]] = None,
+        charge_per: str = 'row',
     ) -> ChargeResult:
         """Push data to the default dataset, charging pay-per-event items.
 
@@ -802,6 +803,10 @@ class Actor(metaclass=_ActorMeta):
                 charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
                 only when all of its events were charged; rows the spending limit cannot
                 pay for in full are not stored. Cannot be combined with ``charged_event_name``.
+            charge_per: ``'row'`` (default) charges ``charges`` for every row; ``'batch'``
+                charges them once for the whole push, e.g.
+                ``push_data(page_rows, charges={'document': 1}, charge_per='batch')``:
+                all rows are stored if the charge is accepted, none if it is refused.
 
         Returns:
             ChargeResult describing how many items were charged and pushed.
@@ -833,7 +838,10 @@ class Actor(metaclass=_ActorMeta):
                 charges,
                 True,  # Actor.push_data always targets the default dataset
                 cls._default_dataset.push_data,
+                charge_per=charge_per,
             )
+        if charge_per != 'row':
+            raise ValueError("charge_per needs charges={...}")
         return await push_data_and_charge(
             charging_manager,
             items,

{scrapely_python-0.1.19 → scrapely_python-0.1.21}/src/scrapely/charging.py

@@ -246,6 +246,8 @@ class ChargingManager:
         # grows), so the local budget gates on the run's true remaining budget.
         self._platform_other_usd: Decimal = Decimal('0')
         self._last_remaining_budget_nd: Optional[int] = None
+        # True once the platform reported a remaining budget (its usage is then in the local budget).
+        self._platform_budget_seen = False
         self._charge_lock = RLock()
         self._flush_lock = asyncio.Lock()
         # Serializes budget calculation/reservation with PPE-aware dataset pushes.
@@ -311,6 +313,7 @@ class ChargingManager:
         self._last_remaining_budget_nd = None
         if nd is None or self._max_total_charge_usd is None or nd < 0:
             return
+        self._platform_budget_seen = True
         other = self._max_total_charge_usd - Decimal(nd) / Decimal(1_000_000_000) - self._total_charged_usd
         if other > self._platform_other_usd:
             self._platform_other_usd = other
@@ -1654,9 +1657,15 @@ async def push_data_with_charges(
     charges: Dict[str, int],
     is_default_dataset: bool,
     push_fn: Any,
+    charge_per: str = 'row',
 ) -> ChargeResult:
     """Push rows that each cost several events, atomically per row.
 
+    ``charge_per='batch'``: ``charges`` are charged ONCE for the whole push (e.g. one
+    ``document`` for all of its page rows). If every event is accepted all rows are
+    stored, otherwise none (and nothing stays billed). The prepaid-row balance is
+    neither used nor changed.
+
     ``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
     A row is stored only when every one of its events was charged: the full set
     is charged first, rows the budget cannot fully pay for are not stored, and
@@ -1666,6 +1675,9 @@ async def push_data_with_charges(
     dataset the default-dataset-item event (when priced) is added per row.
     """
     charges = _validate_row_charges(charges)
+    if charge_per not in ('row', 'batch'):
+        raise ValueError("charge_per must be 'row' or 'batch'")
+    batch = charge_per == 'batch'
     rows_in = items if isinstance(items, list) else [items]
     if not rows_in:
         return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
@@ -1688,7 +1700,8 @@ async def push_data_with_charges(
                 resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
             if default_event and default_event not in per_row and \
                     (charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
-                per_row[default_event] = 1
+                # Per stored row, also for a batch.
+                per_row[default_event] = len(rows_in) if batch else 1
         if not per_row:
             # Every listed event is free: a plain (unpaid) push.
             return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
@@ -1699,7 +1712,8 @@ async def push_data_with_charges(
         row_cost = sum(
             (charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
         )
-        rows = len(rows_in)
+        # Units charged for: rows, or the one batch (row_cost is then the batch's cost).
+        rows = 1 if batch else len(rows_in)
         if charging_manager._max_total_charge_usd is not None and row_cost > 0:
             remaining = charging_manager._remaining_budget_usd()
             rows = min(rows, max(0, int(remaining / row_cost)))
@@ -1717,7 +1731,7 @@ async def push_data_with_charges(
                 else:
                     await charging_manager._void_charge(name, op_id, charged)
 
-        for _attempt in range(4):
+        for _attempt in range(6):
             if rows <= 0:
                 break
             ops = []
@@ -1741,10 +1755,25 @@ async def push_data_with_charges(
             if short_to is None:
                 break
             # Some event came back short: void this attempt in full and retry with
-            # the rows that event can pay for (a void restores the budget).
+            # the whole rows the budget covers at the FULL per-row cost. Sizing
+            # from the short event alone ignores the other events of the row, so
+            # the retry came back short again (appstore 64df0a3608d74af88: 20
+            # details -> 14, retry sized 14, app-item short, 0 saved). What this
+            # attempt was accepted is budget the void gives back; with the
+            # platform's remainingBudgetNd the local budget also knows the rest.
+            attempt_spent = sum(
+                (charging_manager._calculate_event_price(name) or Decimal('0')) * Decimal(charged)
+                for name, charged, _op, _unc in ops
+            )
             await undo(ops)
             ops = []
-            rows = short_to  # got < n * rows, so short_to < rows
+            fit = int(attempt_spent / row_cost) if row_cost > 0 else 0
+            if charging_manager._platform_budget_seen and charging_manager._max_total_charge_usd is not None and row_cost > 0:
+                fit = max(fit, int(charging_manager._remaining_budget_usd() / row_cost))
+            # At least one more single-row try: without the platform's remaining
+            # budget the leftover after the short answer is unknown, and the last
+            # affordable row may still fit.
+            rows = min(rows - 1, max(fit, 1))
 
         if rows <= 0 or not ops:
             await undo(ops)
@@ -1757,11 +1786,11 @@ async def push_data_with_charges(
                 event_charge_limit_reached=True,
             )
 
-        kept = rows_in[:rows]
-        if rows < len(rows_in):
+        kept = rows_in if batch else rows_in[:rows]
+        if len(kept) < len(rows_in):
             charging_manager._mark_charge_limit_hit()
             if is_default_dataset:
-                charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
+                charging_manager._note_refused_default_dataset_items(len(rows_in) - len(kept))
         try:
             await _gated_push(push_fn, kept)
         except Exception as push_err:
@@ -1771,8 +1800,8 @@ async def push_data_with_charges(
             raise
         return ChargeResult(
             chargeable_within_limit=charging_manager.compute_chargeable(),
-            charged_count=rows,
-            event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
+            charged_count=len(kept),
+            event_charge_limit_reached=len(kept) < len(rows_in) or charging_manager.is_charge_limit_reached(),
         )
 
 

{scrapely_python-0.1.19 → scrapely_python-0.1.21/src/scrapely_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.19
+Version: 0.1.21
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.19 → scrapely_python-0.1.21}/tests/test_charge_limit_guard.py

@@ -585,3 +585,146 @@ async def test_platform_other_spend_never_shrinks() -> None:
     api.usage = Decimal('0.10')  # a stale or lower report
     await mgr.charge('result', 1)
     assert mgr._platform_other_usd == seen
+
+
+
+# ---- multi-event retry sizing: whole rows at the FULL per-row cost ----
+
+APP_PRICES = {'app-details': Decimal('0.00015'), 'app-item': Decimal('0.00005')}  # row = $0.0002
+
+
+class RemainingMultiApi(MultiApi):
+    """MultiApi that also reports remainingBudgetNd (server #725)."""
+
+    async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **kw: Any) -> Dict[str, Any]:
+        body = await super().post(path, json_data=json_data, headers=headers, **kw)
+        if not path.endswith('/charge/void') and self.max_usd is not None:
+            body = {**body, 'remainingBudgetNd': int((self.max_usd - self.spent) * 1_000_000_000)}
+        return body
+
+
+def _app(server_max: str, remaining: bool = False):
+    api = (RemainingMultiApi if remaining else MultiApi)(APP_PRICES, Decimal(server_max))
+    return api, _manager(api, prices=dict(APP_PRICES), max_usd='1')
+
+
+@pytest.mark.asyncio
+async def test_retry_after_one_short_event_is_sized_by_the_full_row_cost() -> None:
+    """appstore 64df0a3608d74af88: details 20 -> 14 accepted; the retry must not size from 14."""
+    api, mgr = _app('0.0021')
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(20)], {'app-details': 1, 'app-item': 1}, True, store.push)
+    assert r.charged_count == 10  # $0.0021 / $0.0002 per full row
+    assert len(store.items) == 10
+    assert api.spent == Decimal('0.0020')
+    assert sorted(api.ops.values()) == [('app-details', 10), ('app-item', 10)]
+
+
+@pytest.mark.asyncio
+async def test_retry_when_several_events_come_back_short() -> None:
+    """Both events of the row run short on the way down; it still converges on whole paid rows."""
+    api, mgr = _multi('1', server_max='0.0405')  # 2.53 rows of $0.016 (1 result + 3 pages)
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(6)], {'result': 1, 'page': 3}, True, store.push)
+    assert r.charged_count == 2
+    assert len(store.items) == 2
+    assert api.spent == Decimal('0.032')
+
+
+@pytest.mark.asyncio
+async def test_last_affordable_row_is_saved_without_remaining_budget() -> None:
+    api, mgr = _app('0.0002')  # exactly one row; the short answer hides the leftover
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(5)], {'app-details': 1, 'app-item': 1}, True, store.push)
+    assert r.charged_count == 1 and len(store.items) == 1
+    assert api.spent == Decimal('0.0002')
+
+
+@pytest.mark.asyncio
+async def test_platform_remaining_budget_sizes_the_retry_exactly() -> None:
+    api, mgr = _app('0.00215', remaining=True)  # 10.75 rows
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(20)], {'app-details': 1, 'app-item': 1}, True, store.push)
+    assert r.charged_count == 10 and len(store.items) == 10
+    assert api.spent == Decimal('0.0020')
+
+
+# ---- batch-charged push: charges={'document': 1}, charge_per='batch' ----
+
+DOC = {'document': Decimal('0.01')}
+
+
+def _doc(max_usd: str, server_max: Optional[str] = None, remaining: bool = False):
+    api = (RemainingMultiApi if remaining else MultiApi)(DOC, Decimal(server_max or max_usd))
+    return api, _manager(api, prices=dict(DOC), max_usd=max_usd)
+
+
+def _pages(n: int):
+    return [{'page': i} for i in range(n)]
+
+
+@pytest.mark.asyncio
+async def test_batch_accepted_stores_every_row_for_one_charge() -> None:
+    api, mgr = _doc('1')
+    store = Store()
+    r = await push_data_with_charges(mgr, _pages(7), {'document': 1}, True, store.push, charge_per='batch')
+    assert r.charged_count == 7 and r.event_charge_limit_reached is False
+    assert len(store.items) == 7
+    assert api.spent == Decimal('0.01')
+    assert list(api.ops.values()) == [('document', 1)]
+    assert mgr._prepaid_row_balance == 0, 'batch push neither uses nor adds prepaid rows'
+
+
+@pytest.mark.asyncio
+async def test_batch_refused_stores_nothing() -> None:
+    api, mgr = _doc('0.005')
+    store = Store()
+    r = await push_data_with_charges(mgr, _pages(4), {'document': 1}, True, store.push, charge_per='batch')
+    assert r.charged_count == 0 and r.event_charge_limit_reached is True
+    assert store.items == [] and api.spent == 0
+
+
+@pytest.mark.asyncio
+async def test_batch_at_the_exact_budget_then_the_next_is_refused() -> None:
+    api, mgr = _doc('0.02')
+    store = Store()
+    for _ in range(2):
+        r = await push_data_with_charges(mgr, _pages(3), {'document': 1}, True, store.push, charge_per='batch')
+        assert r.charged_count == 3
+    assert mgr.is_charge_limit_reached() is True
+    r = await push_data_with_charges(mgr, _pages(3), {'document': 1}, True, store.push, charge_per='batch')
+    assert r.charged_count == 0
+    assert len(store.items) == 6 and api.spent == Decimal('0.02')
+
+
+@pytest.mark.asyncio
+async def test_batch_respects_the_usage_aware_budget() -> None:
+    # Local cap $1, but the platform (usage) only has $0.015 left: first batch fits, the second is refused
+    # and its partial charge does not leave rows stored.
+    api, mgr = _doc('1', server_max='0.015', remaining=True)
+    store = Store()
+    r1 = await push_data_with_charges(mgr, _pages(2), {'document': 1}, True, store.push, charge_per='batch')
+    assert r1.charged_count == 2
+    assert mgr._platform_budget_seen is True
+    r2 = await push_data_with_charges(mgr, _pages(2), {'document': 1}, True, store.push, charge_per='batch')
+    assert r2.charged_count == 0
+    assert len(store.items) == 2 and api.spent == Decimal('0.01')
+
+
+@pytest.mark.asyncio
+async def test_batch_with_the_default_dataset_item_event_charges_it_per_row() -> None:
+    prices = {'document': Decimal('0.01'), 'apify-default-dataset-item': Decimal('0.001')}
+    api = MultiApi(prices, Decimal('1'))
+    mgr = _manager(api, prices=dict(prices), max_usd='1')
+    store = Store()
+    r = await push_data_with_charges(mgr, _pages(5), {'document': 1}, True, store.push, charge_per='batch')
+    assert r.charged_count == 5
+    assert api.spent == Decimal('0.01')  # the platform derives the dataset-item event itself
+    assert mgr.get_charged_events().get('apify-default-dataset-item') == 5, 'counted per stored row locally'
+
+
+def test_charge_per_validation() -> None:
+    import asyncio
+    api, mgr = _doc('1')
+    with pytest.raises(ValueError):
+        asyncio.run(push_data_with_charges(mgr, _pages(1), {'document': 1}, True, Store().push, charge_per='page'))