scrapely-python 0.1.17__tar.gz → 0.1.18__tar.gz

{scrapely_python-0.1.17/src/scrapely_python.egg-info → scrapely_python-0.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.17
+Version: 0.1.18
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.17 → scrapely_python-0.1.18}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.17"
+version = "0.1.18"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.17 → scrapely_python-0.1.18}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.17'
+__version__ = '0.1.18'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.17 → scrapely_python-0.1.18}/src/scrapely/actor.py

@@ -28,6 +28,7 @@ from .charging import (
     ChargingManager,
     is_default_dataset_item_event,
     push_data_and_charge,
+    push_data_with_charges,
     set_active_charging_manager,
 )
 from .configuration import Configuration
@@ -781,6 +782,7 @@ class Actor(metaclass=_ActorMeta):
         charged_event_name: Optional[str] = None,
         *,
         event_name: Optional[str] = None,
+        charges: Optional[Dict[str, int]] = None,
     ) -> ChargeResult:
         """Push data to the default dataset, charging pay-per-event items.
 
@@ -796,6 +798,10 @@ class Actor(metaclass=_ActorMeta):
                 (Apify Python SDK name; also accepted positionally).
             event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
                 kept so existing callers keep working.
+            charges: Several events per row, charged atomically: ``{'result': 1, 'page': 3}``
+                charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
+                only when all of its events were charged; rows the spending limit cannot
+                pay for in full are not stored. Cannot be combined with ``charged_event_name``.
 
         Returns:
             ChargeResult describing how many items were charged and pushed.
@@ -818,6 +824,16 @@ class Actor(metaclass=_ActorMeta):
 
         items = data if isinstance(data, list) else [data]
         charging_manager = cls.get_charging_manager()
+        if charges is not None:
+            if event_name is not None:
+                raise ValueError('Pass either charged_event_name or charges, not both')
+            return await push_data_with_charges(
+                charging_manager,
+                items,
+                charges,
+                True,  # Actor.push_data always targets the default dataset
+                cls._default_dataset.push_data,
+            )
         return await push_data_and_charge(
             charging_manager,
             items,

{scrapely_python-0.1.17 → scrapely_python-0.1.18}/src/scrapely/charging.py

@@ -1585,6 +1585,166 @@ async def push_data_and_charge(
         )
 
 
+def _validate_row_charges(charges: Any) -> Dict[str, int]:
+    if not isinstance(charges, dict) or not charges:
+        raise ValueError('charges must be a non-empty dict of {event_name: count_per_row}')
+    out: Dict[str, int] = {}
+    for name, count in charges.items():
+        if not isinstance(name, str) or not name:
+            raise ValueError('charges keys must be event names')
+        if is_default_dataset_item_event(name):
+            raise ValueError(f"Cannot charge for synthetic event '{name}' manually")
+        if not isinstance(count, int) or isinstance(count, bool) or count < 1:
+            raise ValueError(f'charges[{name!r}] must be a positive integer (events per row)')
+        out[name] = count
+    return out
+
+
+async def push_data_with_charges(
+    charging_manager: ChargingManager,
+    items: Union[Dict[str, Any], List[Dict[str, Any]]],
+    charges: Dict[str, int],
+    is_default_dataset: bool,
+    push_fn: Any,
+) -> ChargeResult:
+    """Push rows that each cost several events, atomically per row.
+
+    ``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
+    A row is stored only when every one of its events was charged: the full set
+    is charged first, rows the budget cannot fully pay for are not stored, and
+    any operation that came back short is voided and re-charged for the rows
+    that are kept, so no row is ever stored partly paid and no event is billed
+    for a row that was not stored. Unpriced events are ignored. On the default
+    dataset the default-dataset-item event (when priced) is added per row.
+    """
+    charges = _validate_row_charges(charges)
+    rows_in = items if isinstance(items, list) else [items]
+    if not rows_in:
+        return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
+
+    async with charging_manager._operation_lock:
+        if charging_manager._closed:
+            raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
+        voids_confirmed = await charging_manager._retry_pending_voids()
+
+        if charging_manager._pricing_info is None:
+            await _gated_push(push_fn, rows_in)
+            return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
+
+        per_row: Dict[str, int] = {
+            name: n for name, n in charges.items()
+            if (charging_manager._calculate_event_price(name) or Decimal('0')) > 0
+        }
+        if is_default_dataset:
+            default_event = charging_manager._config.get('default_dataset_item_event_name') or \
+                resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
+            if default_event and default_event not in per_row and \
+                    (charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
+                per_row[default_event] = 1
+        if not per_row:
+            # Every listed event is free: a plain (unpaid) push.
+            return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
+        if not voids_confirmed and any(not is_default_dataset_item_event(n) for n in per_row):
+            raise RuntimeError('An earlier charge for a failed dataset push could not be voided yet; retry the push')
+
+        # Rows the budget can pay for in full (no one-event overdraw: a row is all or nothing).
+        row_cost = sum(
+            (charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
+        )
+        rows = len(rows_in)
+        if charging_manager._max_total_charge_usd is not None and row_cost > 0:
+            remaining = charging_manager._max_total_charge_usd - charging_manager._total_charged_usd
+            rows = min(rows, max(0, int(remaining / row_cost)))
+
+        ops: List[tuple] = []  # (event, charged, op_id, uncertain)
+
+        async def undo(entries: List[tuple]) -> None:
+            for name, charged, op_id, uncertain in entries:
+                if charged <= 0:
+                    continue
+                if op_id is None:
+                    charging_manager._unwind_reservation(name, charged)
+                elif uncertain:
+                    await charging_manager._cancel_uncertain_charge(name, op_id, charged)
+                else:
+                    await charging_manager._void_charge(name, op_id, charged)
+
+        for _attempt in range(4):
+            if rows <= 0:
+                break
+            ops = []
+            short_to: Optional[int] = None
+            for name, n in sorted(per_row.items(), key=lambda kv: (is_default_dataset_item_event(kv[0]), kv[0])):
+                want = n * rows
+                charging_manager._take_last_accepted_operation_id()
+                try:
+                    r = await charging_manager._charge_exclusive(name, want)
+                    op_id = charging_manager._take_last_accepted_operation_id()
+                    ops.append((name, r.charged_count, op_id, False))
+                    got = r.charged_count
+                except ChargeUncertainError as unconfirmed:
+                    # Not confirmed: the intent stays pending and is re-posted at exit,
+                    # so it counts as accepted (items stored, billed once).
+                    ops.append((name, unconfirmed.count, unconfirmed.operation_id, True))
+                    got = unconfirmed.count
+                if got < want:
+                    short_to = got // n
+                    break
+            if short_to is None:
+                break
+            # Some event came back short: void this attempt in full and retry with
+            # the rows that event can pay for (a void restores the budget).
+            await undo(ops)
+            ops = []
+            rows = short_to  # got < n * rows, so short_to < rows
+
+        if rows <= 0 or not ops:
+            await undo(ops)
+            charging_manager._mark_charge_limit_hit()
+            if is_default_dataset:
+                charging_manager._note_refused_default_dataset_items(len(rows_in))
+            return ChargeResult(
+                chargeable_within_limit=charging_manager.compute_chargeable(),
+                charged_count=0,
+                event_charge_limit_reached=True,
+            )
+
+        kept = rows_in[:rows]
+        if rows < len(rows_in):
+            charging_manager._mark_charge_limit_hit()
+            if is_default_dataset:
+                charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
+        try:
+            await _gated_push(push_fn, kept)
+        except Exception as push_err:
+            logger.error('Dataset push failed after charges accepted (rows=%s): %s; voiding the charges', rows, push_err)
+            if _push_certainly_failed(push_err):
+                await undo(ops)
+            raise
+        return ChargeResult(
+            chargeable_within_limit=charging_manager.compute_chargeable(),
+            charged_count=rows,
+            event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
+        )
+
+
+async def _unpaid_push_locked(
+    charging_manager: ChargingManager, rows: List[Dict[str, Any]], is_default_dataset: bool, push_fn: Any
+) -> ChargeResult:
+    """A push whose rows carry no priced event (caller holds the operation lock)."""
+    past_limit = charging_manager.is_charge_limit_reached()
+    allowed = charging_manager._take_unpaid_rows(len(rows)) if is_default_dataset else len(rows)
+    if allowed < len(rows):
+        charging_manager._note_refused_default_dataset_items(len(rows) - allowed)
+    if allowed > 0:
+        await _gated_push(push_fn, rows[:allowed])
+    return ChargeResult(
+        chargeable_within_limit=charging_manager.compute_chargeable(),
+        charged_count=0,
+        event_charge_limit_reached=past_limit,
+    )
+
+
 def _push_is_paid_per_item(
     charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
 ) -> bool:

{scrapely_python-0.1.17 → scrapely_python-0.1.18/src/scrapely_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.17
+Version: 0.1.18
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.17 → scrapely_python-0.1.18}/tests/test_charge_limit_guard.py

@@ -400,3 +400,112 @@ async def test_refused_charge_402_then_plain_push_is_not_saved() -> None:
         await _plain_push(mgr, store, [{'n': n}])
 
     assert [i['n'] for i in store.items] == [0, 1]
+
+
+# ---- atomic multi-event push: push_data(row, charges={...}) ----
+
+from scrapely.charging import push_data_with_charges  # noqa: E402
+
+
+class MultiApi(FakeChargeApi):
+    """Charge endpoint priced per event, with void support (by Idempotency-Key)."""
+
+    def __init__(self, prices: Dict[str, Decimal], max_usd: Optional[Decimal]) -> None:
+        super().__init__(price=Decimal('0'), max_usd=max_usd)
+        self.prices = prices
+        self.ops: Dict[str, tuple] = {}
+        self.voids: List[str] = []
+
+    async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **_: Any) -> Dict[str, Any]:
+        key = (headers or {}).get('Idempotency-Key', '')
+        if path.endswith('/charge/void'):
+            name, accepted = self.ops.pop(key)
+            self.spent -= self.prices[name] * accepted
+            self.voids.append(name)
+            return {}
+        name = (json_data or {}).get('eventName')
+        count = int((json_data or {}).get('count', 1))
+        price = self.prices[name]
+        accepted = count
+        if self.max_usd is not None and price > 0:
+            affordable = int((self.max_usd - self.spent) / price)
+            if affordable < 1:
+                raise HttpError(402, 'charge-limit-exceeded')
+            accepted = min(count, affordable)
+        self.spent += price * accepted
+        self.ops[key] = (name, accepted)
+        return {'acceptedCount': accepted, 'requestedCount': count}
+
+
+PRICES = {'result': Decimal('0.01'), 'page': Decimal('0.002')}  # one row = 1 result + 3 pages = $0.016
+
+
+def _multi(max_usd: str, server_max: Optional[str] = None):
+    api = MultiApi(PRICES, Decimal(server_max or max_usd))
+    return api, _manager(api, prices=dict(PRICES), max_usd=max_usd)
+
+
+@pytest.mark.asyncio
+async def test_multi_event_exact_budget_stores_every_paid_row() -> None:
+    api, mgr = _multi('0.048')  # exactly 3 rows
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
+    assert r.charged_count == 3
+    assert [i['n'] for i in store.items] == [0, 1, 2]
+    assert api.spent == Decimal('0.048')
+    assert mgr.is_charge_limit_reached() is True
+    again = await push_data_with_charges(mgr, [{'n': 9}], {'result': 1, 'page': 3}, True, store.push)
+    assert again.charged_count == 0 and again.event_charge_limit_reached is True
+    assert len(store.items) == 3 and api.spent == Decimal('0.048')
+
+
+@pytest.mark.asyncio
+async def test_multi_event_partial_budget_stores_only_fully_paid_rows() -> None:
+    api, mgr = _multi('0.040')  # 2.5 rows: only 2 fully paid
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(5)], {'result': 1, 'page': 3}, True, store.push)
+    assert r.charged_count == 2 and r.event_charge_limit_reached is True
+    assert [i['n'] for i in store.items] == [0, 1]
+    assert api.spent == Decimal('0.032')  # never a part-paid row
+
+
+@pytest.mark.asyncio
+async def test_multi_event_server_cap_short_voids_and_recharges() -> None:
+    """Local budget says 3 rows; the platform (usage ate the budget) caps 'result' at 2."""
+    api, mgr = _multi('1', server_max='0.032')
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
+    # The platform's remaining budget is only learnt from the short answer, so the
+    # retry keeps what that answer proves affordable: fewer rows, never a part-paid one.
+    assert 1 <= r.charged_count <= 2
+    assert len(store.items) == r.charged_count
+    assert api.spent == Decimal('0.016') * r.charged_count
+    assert api.voids  # the short attempt was voided, not left part-paid
+    assert sorted(n for n, _ in api.ops.values()) == ['page', 'result']
+
+
+@pytest.mark.asyncio
+async def test_multi_event_refusal_stores_nothing_and_bills_nothing() -> None:
+    api, mgr = _multi('0.015')  # less than one full row
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push)
+    assert r.charged_count == 0 and r.event_charge_limit_reached is True
+    assert store.items == [] and api.spent == Decimal('0')
+
+
+@pytest.mark.asyncio
+async def test_multi_event_push_does_not_touch_prepaid_balance() -> None:
+    api, mgr = _multi('0.026')
+    store = Store()
+    await mgr.charge('result', 1)  # prepays one plain row
+    await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push)  # $0.016, paid itself
+    assert mgr._prepaid_row_balance == 1
+    await _plain_push(mgr, store, [{'n': 2}])  # the prepaid row
+    assert [i['n'] for i in store.items] == [1, 2]
+
+
+def test_multi_event_validation() -> None:
+    from scrapely.charging import _validate_row_charges
+    for bad in ({}, {'result': 0}, {'result': 1.5}, {'apify-default-dataset-item': 1}, {'': 1}, None):
+        with pytest.raises(ValueError):
+            _validate_row_charges(bad)