scrapely-python 0.1.9__tar.gz → 0.1.11__tar.gz

{scrapely_python-0.1.9/src/scrapely_python.egg-info → scrapely_python-0.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.9
+Version: 0.1.11
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.9 → scrapely_python-0.1.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.9"
+version = "0.1.11"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.9 → scrapely_python-0.1.11}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.9'
+__version__ = '0.1.11'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.9 → scrapely_python-0.1.11}/src/scrapely/actor.py

@@ -24,7 +24,12 @@ from crawlee.configuration import Configuration as CrawleeConfiguration
 from crawlee.storage_clients._base import StorageClient
 from crawlee.storages import Dataset, KeyValueStore, RequestQueue
 
-from .charging import ChargingManager, is_default_dataset_item_event, push_data_and_charge
+from .charging import (
+    ChargingManager,
+    is_default_dataset_item_event,
+    push_data_and_charge,
+    set_active_charging_manager,
+)
 from .configuration import Configuration
 from .crypto import decrypt_secret_inputs
 from .proxy_configuration import with_proxy_key_aliases
@@ -515,6 +520,7 @@ class Actor(metaclass=_ActorMeta):
             # Clear charging + patched client so a second Actor lifecycle cannot
             # post charges to the previous run_id / credentials.
             cls._charging_manager = None
+            set_active_charging_manager(None)
             cls._apify_client_instance = None
 
             cls._default_dataset = None
@@ -1222,6 +1228,8 @@ class Actor(metaclass=_ActorMeta):
             # Prefer the storage factory client (same URL/token/run as API storage).
             client = cls._client if cls._client is not None else cls.new_client()
             cls._charging_manager = ChargingManager(config, client)
+            # Storage clients refuse default-dataset writes past this run's limit.
+            set_active_charging_manager(cls._charging_manager)
         return cls._charging_manager
 
 

{scrapely_python-0.1.9 → scrapely_python-0.1.11}/src/scrapely/charging.py

@@ -9,7 +9,9 @@ never treated as retries of a previous operation.
 from __future__ import annotations
 
 import asyncio
+import contextvars
 import json
+import os
 import logging
 import time
 import uuid
@@ -63,6 +65,50 @@ _RETRY_BUDGET_SECS = 60.0
 LOCAL_CHARGING_LOG_DATASET_NAME = 'charging_log'
 
 
+# Charge-limit safety net for the default dataset.
+#
+# Once a pay-per-event run has used up its spending limit (maxTotalChargeUsd),
+# rows written to the run's default dataset would go out unbilled. The SDK then
+# refuses such writes: nothing is stored, one warning is logged and
+# ``ChargingManager.is_charge_limit_reached()`` turns True. Pushes made by
+# ``push_data_and_charge`` (``Actor.push_data``) are decided there and marked
+# with this context variable, so the storage client does not check them twice.
+_charge_gated_push: contextvars.ContextVar[bool] = contextvars.ContextVar(
+    'scrapely_charge_gated_push', default=False
+)
+
+# The charging manager of the running Actor (set by ``Actor``); storage clients
+# consult it before writing to the default dataset.
+_active_charging_manager: Optional['ChargingManager'] = None
+
+
+def set_active_charging_manager(manager: Optional['ChargingManager']) -> None:
+    """Register the charging manager whose limit guards default-dataset writes."""
+    global _active_charging_manager
+    _active_charging_manager = manager
+
+
+def default_dataset_id_from_env() -> Optional[str]:
+    """The run's default dataset ID, as the platform passes it."""
+    return os.environ.get('ACTOR_DEFAULT_DATASET_ID') or os.environ.get('APIFY_DEFAULT_DATASET_ID')
+
+
+def refuse_default_dataset_push(item_count: int) -> bool:
+    """True when a write of ``item_count`` rows to the default dataset must be dropped.
+
+    Called by storage clients for writes that did not go through
+    ``push_data_and_charge`` (``Actor.open_dataset().push_data()``, crawler
+    ``context.push_data``, ...). Non-PPE runs and runs below their limit are
+    never refused.
+    """
+    if _charge_gated_push.get():
+        return False
+    manager = _active_charging_manager
+    if manager is None:
+        return False
+    return manager._refuse_default_dataset_push(item_count)
+
+
 def is_default_dataset_item_event(event_name: str) -> bool:
     """Check if event name is a default dataset item event."""
     return event_name in (APIFY_DEFAULT_DATASET_ITEM_EVENT, SCRAPELY_DEFAULT_DATASET_ITEM_EVENT)
@@ -209,10 +255,69 @@ class ChargingManager:
         self._last_accepted_operation_id: Optional[str] = None
         # Voids not yet confirmed by the platform; retried before the next push and at exit.
         self._pending_voids: List[tuple[str, str, int, bool]] = []
+        # Sticky: the platform refused or capped a charge (HTTP 402 / fewer
+        # events accepted) or the local budget gate trimmed one.
+        self._charge_limit_hit = False
+        # Default-dataset rows refused after the limit (one warning, then a count).
+        self._refused_default_dataset_items = 0
 
     def close(self) -> None:
         """Mark the manager closed; subsequent charge/push operations are rejected."""
         self._closed = True
+        if self._refused_default_dataset_items:
+            logger.warning(
+                '%s result(s) pushed to the default dataset after the spending limit was '
+                'reached were not saved',
+                self._refused_default_dataset_items,
+            )
+
+    def is_charge_limit_reached(self) -> bool:
+        """Whether this pay-per-event run has used up its spending limit.
+
+        True once a charge was refused or capped by the limit (the platform's
+        HTTP 402 or a partial acceptance, or the local budget gate), or once the
+        remaining budget cannot pay for even one event of the cheapest priced
+        type. From then on, pushes to the default dataset that are not paid
+        for are refused. Always False for actors without pay-per-event
+        pricing, without a limit, or with only free events.
+        """
+        if self._pricing_info is None:
+            return False
+        if self._charge_limit_hit:
+            return True
+        if self._max_total_charge_usd is None:
+            return False
+        prices = [price for price in self._per_event_prices.values() if price > 0]
+        if not prices:
+            return False
+        return self._max_total_charge_usd - self._total_charged_usd < min(prices)
+
+    def _mark_charge_limit_hit(self) -> None:
+        if self._pricing_info is not None:
+            self._charge_limit_hit = True
+
+    def _note_refused_default_dataset_items(self, count: int) -> None:
+        """Count rows dropped because of the limit; warn on the first one only."""
+        if count <= 0:
+            return
+        if not self._refused_default_dataset_items:
+            limit = (
+                f'${self._max_total_charge_usd}' if self._max_total_charge_usd is not None else 'the limit'
+            )
+            logger.warning(
+                'The spending limit of this run (maxTotalChargeUsd = %s) has been reached: '
+                'results pushed to the default dataset are no longer saved. The Actor should '
+                'stop now (check ChargeResult.event_charge_limit_reached or '
+                'Actor.get_charging_manager().is_charge_limit_reached()).',
+                limit,
+            )
+        self._refused_default_dataset_items += count
+
+    def _refuse_default_dataset_push(self, item_count: int) -> bool:
+        if item_count <= 0 or not self.is_charge_limit_reached():
+            return False
+        self._note_refused_default_dataset_items(item_count)
+        return True
 
     @property
     def charge_lock(self) -> RLock:
@@ -688,6 +793,8 @@ class ChargingManager:
                             deadline=deadline,
                         )
                     except ChargeRejectedError as err:
+                        if _is_charge_limit_rejection(err):
+                            self._mark_charge_limit_hit()
                         self._unwind_reservation(event_name, count)
                         self._drop_pending(idempotency_key)
                         logger.warning(
@@ -699,6 +806,7 @@ class ChargingManager:
                         continue
                     if accepted is not None:
                         if accepted < count:
+                            self._mark_charge_limit_hit()
                             self._unwind_reservation(event_name, count - accepted)
                         self._drop_pending(idempotency_key)
                         confirmed += 1
@@ -784,6 +892,9 @@ class ChargingManager:
                         else 0
                     )
 
+            if chargeable_count < count:
+                self._mark_charge_limit_hit()
+
             if chargeable_count <= 0:
                 return ChargeResult(
                     chargeable_within_limit=self._compute_chargeable(),
@@ -835,7 +946,9 @@ class ChargingManager:
                     count=chargeable_count,
                     idempotency_key=idempotency_key,
                 )
-            except ChargeRejectedError:
+            except ChargeRejectedError as rejected:
+                if _is_charge_limit_rejection(rejected):
+                    self._mark_charge_limit_hit()
                 # Server definitively rejected: unwind local reservation.
                 self._drop_pending(idempotency_key)
                 self._unwind_reservation(event_name, chargeable_count)
@@ -849,6 +962,8 @@ class ChargingManager:
                 if accepted > 0:
                     self._last_accepted_operation_id = idempotency_key
                 if accepted < chargeable_count:
+                    # The platform capped the charge at the run's limit.
+                    self._mark_charge_limit_hit()
                     self._unwind_reservation(event_name, chargeable_count - accepted)
                     chargeable_count = accepted
                     result = ChargeResult(
@@ -1066,6 +1181,15 @@ class ChargingManager:
 
 
 
+def _is_charge_limit_rejection(err: ChargeRejectedError) -> bool:
+    """Whether a rejected charge was refused because the run's limit is used up."""
+    return (
+        err.status_code == 402
+        or err.reason == 'charge-limit-exceeded'
+        or 'charge-limit-exceeded' in str(err).lower()
+    )
+
+
 def _read_accepted_count(body: Any, fallback: int) -> int:
     if not isinstance(body, dict):
         return fallback
@@ -1166,6 +1290,24 @@ async def push_data_and_charge(
             if fallback_event:
                 resolved_event = fallback_event
 
+        # Charge-limit safety net: once the run's spending limit is used up, a
+        # default-dataset push that does not pay per row (no event, or only
+        # free events) stores nothing. Paid pushes go on below, where the
+        # budget cap stores only the rows that are actually charged.
+        if (
+            is_ppe
+            and is_default_dataset
+            and items
+            and charging_manager.is_charge_limit_reached()
+            and not _push_is_paid_per_item(charging_manager, event_name, resolved_event)
+        ):
+            charging_manager._note_refused_default_dataset_items(len(items))
+            return ChargeResult(
+                chargeable_within_limit=charging_manager.compute_chargeable(),
+                charged_count=0,
+                event_charge_limit_reached=True,
+            )
+
         limited_count = len(items)
         if is_ppe and resolved_event:
             # Keep the explicit event separate from the implicit default-dataset
@@ -1177,6 +1319,10 @@ async def push_data_and_charge(
             )
 
         limited_items = items[:limited_count] if limited_count < len(items) else items
+        if is_ppe and is_default_dataset and limited_count < len(items):
+            # Rows beyond what the budget can pay for are dropped.
+            charging_manager._mark_charge_limit_hit()
+            charging_manager._note_refused_default_dataset_items(len(items) - limited_count)
 
         if not limited_items:
             return ChargeResult(
@@ -1302,9 +1448,13 @@ async def push_data_and_charge(
                             await charging_manager._cancel_uncertain_charge(name, op_id, reserved)
                         else:
                             await charging_manager._void_charge(name, op_id, reserved)
+                if is_default_dataset and accepted_for_items < len(limited_items):
+                    charging_manager._note_refused_default_dataset_items(
+                        len(limited_items) - accepted_for_items
+                    )
                 if accepted_for_items > 0:
                     try:
-                        await push_fn(limited_items[:accepted_for_items])
+                        await _gated_push(push_fn, limited_items[:accepted_for_items])
                     except Exception as push_err:
                         # A result is billed only if it was saved: void this
                         # operation's platform charges and unwind local
@@ -1333,7 +1483,9 @@ async def push_data_and_charge(
                         raise
                 return ChargeResult(
                     chargeable_within_limit=charging_manager.compute_chargeable(),
-                    charged_count=accepted_for_items * len(results) if results else 0,
+                    # Rows charged, as Apify reports it for the pushed event; the
+                    # implicit default-dataset event is not counted on top.
+                    charged_count=accepted_for_items,
                     event_charge_limit_reached=any(
                         result.event_charge_limit_reached for result in results
                     )
@@ -1342,7 +1494,7 @@ async def push_data_and_charge(
 
 
         # No billable events — still persist the budget-limited items.
-        await push_fn(limited_items)
+        await _gated_push(push_fn, limited_items)
         return ChargeResult(
             chargeable_within_limit=charging_manager.compute_chargeable(),
             charged_count=0,
@@ -1350,6 +1502,23 @@ async def push_data_and_charge(
         )
 
 
+def _push_is_paid_per_item(
+    charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
+) -> bool:
+    """Whether every row of this push is charged a positive price."""
+    names = {name for name in (event_name, resolved_event) if name}
+    return any((charging_manager._calculate_event_price(name) or Decimal('0')) > 0 for name in names)
+
+
+async def _gated_push(push_fn: Any, items: List[Dict[str, Any]]) -> Any:
+    """Run a push already decided by push_data_and_charge (storage skips its own check)."""
+    token = _charge_gated_push.set(True)
+    try:
+        return await push_fn(items)
+    finally:
+        _charge_gated_push.reset(token)
+
+
 def _http_status_of(err: BaseException) -> Optional[int]:
     status = getattr(err, 'status_code', None)
     if status is None:

{scrapely_python-0.1.9 → scrapely_python-0.1.11}/src/scrapely/charging_client.py

@@ -43,7 +43,11 @@ class _ChargingManagerHooks:
         from .charging import has_default_dataset_item_event_in_pricing
 
         pricing_info = self._charging_manager.get_pricing_info()
-        return has_default_dataset_item_event_in_pricing(pricing_info.per_event_prices or {})
+        # Past the spending limit, route default-dataset pushes through
+        # push_data_and_charge too, which refuses rows nobody pays for.
+        return has_default_dataset_item_event_in_pricing(
+            pricing_info.per_event_prices or {}
+        ) or bool(getattr(self._charging_manager, 'is_charge_limit_reached', lambda: False)())
 
     async def push_and_charge(
         self,

{scrapely_python-0.1.9 → scrapely_python-0.1.11}/src/scrapely/storage/scrapely_dataset_client.py

@@ -99,6 +99,12 @@ class ScrapelyDatasetClient(DatasetClient):
             items = [data]
         else:
             items = list(data)
+        # Charge-limit safety net: a pay-per-event run past its spending limit
+        # must not store unbilled rows in its default dataset.
+        from ..charging import default_dataset_id_from_env, refuse_default_dataset_push
+
+        if self._metadata.id == default_dataset_id_from_env() and refuse_default_dataset_push(len(items)):
+            return
         await self._api.dataset_push_items(self._metadata.id, items)
 
     @override

{scrapely_python-0.1.9 → scrapely_python-0.1.11/src/scrapely_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.9
+Version: 0.1.11
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.9 → scrapely_python-0.1.11}/src/scrapely_python.egg-info/SOURCES.txt

@@ -52,6 +52,7 @@ tests/test_api_client_retries.py
 tests/test_apify_parity.py
 tests/test_browser_configuration.py
 tests/test_cdp_retry.py
+tests/test_charge_limit_guard.py
 tests/test_charge_retries.py
 tests/test_charging_parity.py
 tests/test_charging_pending.py

scrapely_python-0.1.11/tests/test_charge_limit_guard.py

@@ -0,0 +1,291 @@
+"""Charge-limit safety net: no unbilled rows in the default dataset past maxTotalChargeUsd."""
+
+from __future__ import annotations
+
+import logging
+from decimal import Decimal
+from typing import Any, Dict, List, Optional
+from unittest.mock import MagicMock
+
+import pytest
+
+from scrapely.charging import (
+    ChargingManager,
+    push_data_and_charge,
+    refuse_default_dataset_push,
+    set_active_charging_manager,
+)
+from scrapely.charging_client import _ChargingManagerHooks
+from scrapely.pricing_types import PayPerEventActorPricingInfo, PricingPerEvent
+from scrapely.storage._metadata import to_dataset_metadata
+from scrapely.storage.scrapely_dataset_client import ScrapelyDatasetClient
+
+DEFAULT_DATASET_ID = 'aaaaaaaa-0000-4000-8000-000000000001'
+NAMED_DATASET_ID = 'bbbbbbbb-0000-4000-8000-000000000002'
+
+
+class HttpError(Exception):
+    def __init__(self, status_code: int, message: str) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+
+
+class FakeChargeApi:
+    """Charge endpoint with a server-side budget (like the platform ledger)."""
+
+    def __init__(self, *, price: Decimal, max_usd: Optional[Decimal], reject_after: Optional[int] = None) -> None:
+        self.run_id = 'run-guard-1'
+        self.price = price
+        self.max_usd = max_usd
+        self.spent = Decimal('0')
+        self.calls = 0
+        self.reject_after = reject_after
+
+    async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, **_: Any) -> Dict[str, Any]:
+        count = int((json_data or {}).get('count', 1))
+        self.calls += 1
+        if self.reject_after is not None and self.calls > self.reject_after:
+            # Budget used up on the server (for example by platform usage).
+            raise HttpError(402, 'charge-limit-exceeded')
+        accepted = count
+        if self.max_usd is not None and self.price > 0:
+            affordable = int((self.max_usd - self.spent) / self.price)
+            if affordable < 1:
+                raise HttpError(402, 'charge-limit-exceeded')
+            accepted = min(count, affordable)
+        self.spent += self.price * accepted
+        return {'acceptedCount': accepted, 'requestedCount': count}
+
+
+def _manager(
+    api: FakeChargeApi,
+    *,
+    prices: Optional[Dict[str, Decimal]] = None,
+    max_usd: Optional[str] = '0.03',
+    ppe: bool = True,
+) -> ChargingManager:
+    config = MagicMock()
+    config.get = MagicMock(
+        side_effect=lambda key, default=None: {
+            'is_at_home': True,
+            'actor_run_id': api.run_id,
+            'max_total_charge_usd': max_usd,
+        }.get(key, default)
+    )
+    mgr = ChargingManager(config, api)  # type: ignore[arg-type]
+    if ppe:
+        prices = prices if prices is not None else {'result': Decimal('0.01')}
+        mgr._pricing_info = PayPerEventActorPricingInfo(
+            pricing_per_event={name: PricingPerEvent(event_price_usd=p) for name, p in prices.items()}
+        )
+        mgr._per_event_prices = dict(prices)
+    mgr._initialized = True
+    mgr._max_total_charge_usd = Decimal(max_usd) if max_usd else None
+    mgr._POST_ATTEMPTS = 1
+    return mgr
+
+
+class Store:
+    def __init__(self) -> None:
+        self.items: List[Dict[str, Any]] = []
+
+    async def push(self, items: List[Dict[str, Any]]) -> None:
+        self.items.extend(items)
+
+
+async def _plain_push(mgr: ChargingManager, store: Store, rows: List[Dict[str, Any]]):
+    """Actor.push_data(rows) without an event."""
+    return await push_data_and_charge(mgr, rows, None, True, store.push)
+
+
+async def _charged_push(mgr: ChargingManager, store: Store, rows: List[Dict[str, Any]], event: str = 'result'):
+    return await push_data_and_charge(mgr, rows, event, True, store.push)
+
+
+@pytest.fixture(autouse=True)
+def _reset_active_manager(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv('ACTOR_DEFAULT_DATASET_ID', DEFAULT_DATASET_ID)
+    yield
+    set_active_charging_manager(None)
+
+
+def _limit_warnings(caplog: pytest.LogCaptureFixture) -> List[str]:
+    return [r.getMessage() for r in caplog.records if 'spending limit of this run' in r.getMessage()]
+
+
+@pytest.mark.asyncio
+async def test_below_limit_behaves_as_before() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('1'))
+    mgr = _manager(api, max_usd='1')
+    store = Store()
+
+    r1 = await _plain_push(mgr, store, [{'n': 1}, {'n': 2}])
+    r2 = await _charged_push(mgr, store, [{'n': 3}])
+    await mgr.charge('result', 2)
+
+    assert [i['n'] for i in store.items] == [1, 2, 3]
+    assert r1.event_charge_limit_reached is False
+    assert r2.charged_count == 1 and r2.event_charge_limit_reached is False
+    assert mgr.is_charge_limit_reached() is False
+    assert refuse_default_dataset_push(5) is False
+
+
+@pytest.mark.asyncio
+async def test_limit_reached_exactly_by_last_charged_row_refuses_later_free_rows(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
+    mgr = _manager(api)
+    store = Store()
+
+    r = await _charged_push(mgr, store, [{'n': 1}, {'n': 2}, {'n': 3}])
+    assert r.charged_count == 3
+    assert [i['n'] for i in store.items] == [1, 2, 3]
+    assert mgr.is_charge_limit_reached() is True
+
+    # Documented edge case: a later free summary/error row is refused too.
+    with caplog.at_level(logging.WARNING):
+        summary = await _plain_push(mgr, store, [{'summary': True}])
+    assert summary.event_charge_limit_reached is True and summary.charged_count == 0
+    assert len(store.items) == 3
+    assert len(_limit_warnings(caplog)) == 1
+
+
+@pytest.mark.asyncio
+async def test_plain_push_then_separate_charge_stops_at_the_cap(caplog: pytest.LogCaptureFixture) -> None:
+    """The leaking pattern: push without an event, charge separately, ignore the result."""
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
+    mgr = _manager(api)
+    store = Store()
+
+    with caplog.at_level(logging.WARNING):
+        for n in range(10):
+            await _plain_push(mgr, store, [{'n': n}])
+            await mgr.charge('result', 1)
+
+    assert [i['n'] for i in store.items] == [0, 1, 2]
+    assert mgr._refused_default_dataset_items == 7
+    assert len(_limit_warnings(caplog)) == 1
+
+
+@pytest.mark.asyncio
+async def test_platform_402_stops_plain_pushes() -> None:
+    """The server refuses a charge the local budget still allowed (e.g. usage ate the budget)."""
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=None, reject_after=2)
+    mgr = _manager(api, max_usd='5')
+    store = Store()
+
+    for n in range(6):
+        await _plain_push(mgr, store, [{'n': n}])
+        await mgr.charge('result', 1)
+
+    # Row 2 was pushed before its charge came back 402; nothing after that.
+    assert [i['n'] for i in store.items] == [0, 1, 2]
+    assert mgr.is_charge_limit_reached() is True
+
+
+@pytest.mark.asyncio
+async def test_charged_push_after_limit_stores_nothing(caplog: pytest.LogCaptureFixture) -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
+    mgr = _manager(api)
+    store = Store()
+
+    first = await _charged_push(mgr, store, [{'n': n} for n in range(5)])
+    assert first.charged_count == 3 and first.event_charge_limit_reached is True
+    with caplog.at_level(logging.WARNING):
+        again = await _charged_push(mgr, store, [{'n': 9}])
+        free_event = await _charged_push(mgr, store, [{'n': 10}], event='free-event')
+    assert again.charged_count == 0 and again.event_charge_limit_reached is True
+    assert free_event.event_charge_limit_reached is True
+    assert [i['n'] for i in store.items] == [0, 1, 2]
+    assert api.spent == Decimal('0.03')
+    # One warning for the whole run, not one per push.
+    assert len(_limit_warnings(caplog)) <= 1
+
+
+@pytest.mark.asyncio
+async def test_non_ppe_actor_is_unaffected() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
+    mgr = _manager(api, ppe=False, max_usd='0.01')
+    set_active_charging_manager(mgr)
+    store = Store()
+
+    for n in range(5):
+        await _plain_push(mgr, store, [{'n': n}])
+        await mgr.charge('result', 1)
+
+    assert len(store.items) == 5
+    assert mgr.is_charge_limit_reached() is False
+    assert refuse_default_dataset_push(1) is False
+
+
+@pytest.mark.asyncio
+async def test_free_events_only_never_reach_the_limit() -> None:
+    api = FakeChargeApi(price=Decimal('0'), max_usd=Decimal('0.01'))
+    mgr = _manager(api, prices={'free-event': Decimal('0')}, max_usd='0.01')
+    store = Store()
+
+    for n in range(5):
+        await _charged_push(mgr, store, [{'n': n}], event='free-event')
+        await _plain_push(mgr, store, [{'p': n}])
+
+    assert len(store.items) == 10
+    assert mgr.is_charge_limit_reached() is False
+
+
+def _dataset_client(dataset_id: str) -> tuple[ScrapelyDatasetClient, List[Any]]:
+    pushed: List[Any] = []
+    api = MagicMock()
+
+    async def dataset_push_items(ds_id: str, items: List[Any]) -> None:
+        pushed.append((ds_id, list(items)))
+
+    api.dataset_push_items = dataset_push_items
+    return ScrapelyDatasetClient(api=api, metadata=to_dataset_metadata({'id': dataset_id}, dataset_id)), pushed
+
+
+@pytest.mark.asyncio
+async def test_open_dataset_default_push_refused_named_dataset_unaffected() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
+    mgr = _manager(api, max_usd='0.01')
+    set_active_charging_manager(mgr)
+    default_client, default_pushed = _dataset_client(DEFAULT_DATASET_ID)
+    named_client, named_pushed = _dataset_client(NAMED_DATASET_ID)
+
+    # Below the limit both store.
+    await default_client.push_data([{'n': 1}])
+    await named_client.push_data([{'n': 1}])
+    await mgr.charge('result', 1)  # uses the whole budget
+    assert mgr.is_charge_limit_reached() is True
+
+    # Actor.open_dataset().push_data() / crawler pushes to the default dataset: refused.
+    await default_client.push_data([{'n': 2}, {'n': 3}])
+    # A named dataset keeps working.
+    await named_client.push_data([{'n': 2}])
+
+    assert default_pushed == [(DEFAULT_DATASET_ID, [{'n': 1}])]
+    assert named_pushed == [(NAMED_DATASET_ID, [{'n': 1}]), (NAMED_DATASET_ID, [{'n': 2}])]
+
+
+@pytest.mark.asyncio
+async def test_paid_push_through_dataset_client_is_not_checked_twice() -> None:
+    """push_data_and_charge decides; its push reaches the storage client unblocked."""
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.02'))
+    mgr = _manager(api, max_usd='0.02')
+    set_active_charging_manager(mgr)
+    client, pushed = _dataset_client(DEFAULT_DATASET_ID)
+
+    r = await push_data_and_charge(mgr, [{'n': 1}, {'n': 2}], 'result', True, client.push_data)
+    assert r.charged_count == 2
+    assert mgr.is_charge_limit_reached() is True
+    assert pushed == [(DEFAULT_DATASET_ID, [{'n': 1}, {'n': 2}])]
+
+
+@pytest.mark.asyncio
+async def test_apify_client_hooks_intercept_default_dataset_past_limit() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
+    mgr = _manager(api, max_usd='0.01')
+    hooks = _ChargingManagerHooks(mgr)
+    assert hooks.should_charge_dataset_pushes(DEFAULT_DATASET_ID) is False
+    await mgr.charge('result', 1)
+    assert hooks.should_charge_dataset_pushes(DEFAULT_DATASET_ID) is True

{scrapely_python-0.1.9 → scrapely_python-0.1.11}/tests/test_charging_parity.py

@@ -338,3 +338,41 @@ async def test_local_mode_never_posts_charges_despite_client_and_run_id() -> Non
     # Flushing must not attempt any remote POST either.
     assert await mgr.flush_pending() == 0
     assert client.post_calls == []
+
+
+@pytest.mark.asyncio
+async def test_push_data_with_explicit_event_reports_rows_not_rows_times_events() -> None:
+    """charged_count is the number of rows charged, not rows x events.
+
+    With an explicit event on the default dataset the SDK also charges the
+    (unpriced) default-dataset-item event. Apify returns the explicit event's
+    count; summing both reported twice the rows pushed.
+    """
+    from scrapely.charging import push_data_and_charge
+
+    pricing = '{"pricingModel": "PAY_PER_EVENT", "pricingPerEvent": {"actorChargeEvents": {"profile": {"eventPriceUsd": 0.001, "eventTitle": "Profile"}}}}'
+    config = _config_with({
+        'is_at_home': False,
+        'actor_pricing_info': pricing,
+        'charged_event_counts': '{}',
+        'max_total_charge_usd': '10',
+    })
+    mgr = ChargingManager(config, None)
+    await mgr.init()
+
+    pushed: List[Any] = []
+
+    async def push_fn(items: List[Any]) -> None:
+        pushed.extend(items)
+
+    rows = [{'i': i} for i in range(3)]
+    result = await push_data_and_charge(mgr, rows, 'profile', True, push_fn)
+
+    assert len(pushed) == 3
+    assert result.charged_count == 3
+    assert result.event_charge_limit_reached is False
+    assert mgr.get_charged_event_count('profile') == 3
+
+    # Without an explicit event the default event alone is counted once.
+    result2 = await push_data_and_charge(mgr, rows, None, True, push_fn)
+    assert result2.charged_count == 3

{scrapely_python-0.1.9 → scrapely_python-0.1.11}/tests/test_charging_pending.py

@@ -286,7 +286,8 @@ async def test_explicit_and_default_dataset_events_are_each_reserved_once() -> N
     )
 
     assert len(pushed) == 2
-    assert result.charged_count == 4
+    # Rows charged (Apify parity), not rows x events: both events billed 2 each.
+    assert result.charged_count == 2
     assert mgr.get_charged_event_count('page-open') == 2
     assert mgr.get_charged_event_count('apify-default-dataset-item') == 2
     assert mgr.get_total_charged_usd() == Decimal('0.04')