scrapely-python 0.1.16__py3-none-any.whl → 0.1.18__py3-none-any.whl

scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.16'
+__version__ = '0.1.18'
 __all__ = [
     # Core
     'Actor',

scrapely/actor.py

@@ -28,6 +28,7 @@ from .charging import (
     ChargingManager,
     is_default_dataset_item_event,
     push_data_and_charge,
+    push_data_with_charges,
     set_active_charging_manager,
 )
 from .configuration import Configuration
@@ -781,6 +782,7 @@ class Actor(metaclass=_ActorMeta):
         charged_event_name: Optional[str] = None,
         *,
         event_name: Optional[str] = None,
+        charges: Optional[Dict[str, int]] = None,
     ) -> ChargeResult:
         """Push data to the default dataset, charging pay-per-event items.
 
@@ -796,6 +798,10 @@ class Actor(metaclass=_ActorMeta):
                 (Apify Python SDK name; also accepted positionally).
             event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
                 kept so existing callers keep working.
+            charges: Several events per row, charged atomically: ``{'result': 1, 'page': 3}``
+                charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
+                only when all of its events were charged; rows the spending limit cannot
+                pay for in full are not stored. Cannot be combined with ``charged_event_name``.
 
         Returns:
             ChargeResult describing how many items were charged and pushed.
@@ -818,6 +824,16 @@ class Actor(metaclass=_ActorMeta):
 
         items = data if isinstance(data, list) else [data]
         charging_manager = cls.get_charging_manager()
+        if charges is not None:
+            if event_name is not None:
+                raise ValueError('Pass either charged_event_name or charges, not both')
+            return await push_data_with_charges(
+                charging_manager,
+                items,
+                charges,
+                True,  # Actor.push_data always targets the default dataset
+                cls._default_dataset.push_data,
+            )
         return await push_data_and_charge(
             charging_manager,
             items,

scrapely/charging.py

@@ -270,6 +270,13 @@ class ChargingManager:
         self._charge_limit_hit = False
         # Default-dataset rows refused after the limit (one warning, then a count).
         self._refused_default_dataset_items = 0
+        # Prepaid-row balance: priced events charged through charge() minus rows
+        # pushed to the default dataset without a per-row charge. An Actor that
+        # charges first and then pushes plainly (charge('product'); push_data(row))
+        # has a positive balance after the charge, so the row it already paid for
+        # is still saved when that charge used up the spending limit. Push-then-
+        # charge Actors never go positive, so they get nothing extra for free.
+        self._prepaid_row_balance = 0
 
     def close(self) -> None:
         """Mark the manager closed; subsequent charge/push operations are rejected."""
@@ -323,8 +330,40 @@ class ChargingManager:
             )
         self._refused_default_dataset_items += count
 
+    def _note_prepaid_charge(self, event_name: str, charged_count: int) -> None:
+        if (
+            charged_count > 0
+            and not is_default_dataset_item_event(event_name)
+            and self._per_event_prices.get(event_name, 0) > 0
+        ):
+            self._prepaid_row_balance += charged_count
+
+    def _take_unpaid_rows(self, item_count: int) -> int:
+        """Account for ``item_count`` rows pushed without a per-row charge.
+
+        Returns how many of them may be stored: all of them while the limit is
+        not reached, afterwards only the rows already paid for by earlier
+        charges (the positive prepaid balance).
+        """
+        if item_count <= 0:
+            return 0
+        if not self.is_charge_limit_reached():
+            self._prepaid_row_balance -= item_count
+            return item_count
+        allowed = max(0, min(item_count, self._prepaid_row_balance))
+        self._prepaid_row_balance -= allowed
+        return allowed
+
     def _refuse_default_dataset_push(self, item_count: int) -> bool:
-        if item_count <= 0 or not self.is_charge_limit_reached():
+        if item_count <= 0 or self._pricing_info is None:
+            return False
+        if not self.is_charge_limit_reached():
+            self._take_unpaid_rows(item_count)
+            return False
+        # All-or-nothing here (storage clients cannot store a slice): store the
+        # batch only when earlier charges paid for every row of it.
+        if self._prepaid_row_balance >= item_count:
+            self._prepaid_row_balance -= item_count
             return False
         self._note_refused_default_dataset_items(item_count)
         return True
@@ -851,7 +890,9 @@ class ChargingManager:
     async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
         """Atomically reserve and submit one logical charge operation."""
         async with self._operation_lock:
-            return await self._charge_exclusive(event_name, count)
+            result = await self._charge_exclusive(event_name, count)
+            self._note_prepaid_charge(event_name, result.charged_count)
+            return result
 
     async def _charge_exclusive(self, event_name: str, count: int = 1) -> ChargeResult:
         if self._closed:
@@ -1327,19 +1368,27 @@ async def push_data_and_charge(
         # default-dataset push that does not pay per row (no event, or only
         # free events) stores nothing. Paid pushes go on below, where the
         # budget cap stores only the rows that are actually charged.
+        # Rows the Actor already paid for with an earlier charge() (charge first,
+        # then a plain push) are still stored: see ChargingManager._take_unpaid_rows.
+        # The result still reports the limit, so the Actor stops after them.
+        unpaid_past_limit = False
         if (
             is_ppe
             and is_default_dataset
             and items
-            and charging_manager.is_charge_limit_reached()
             and not _push_is_paid_per_item(charging_manager, event_name, resolved_event)
         ):
-            charging_manager._note_refused_default_dataset_items(len(items))
-            return ChargeResult(
-                chargeable_within_limit=charging_manager.compute_chargeable(),
-                charged_count=0,
-                event_charge_limit_reached=True,
-            )
+            unpaid_past_limit = charging_manager.is_charge_limit_reached()
+            allowed = charging_manager._take_unpaid_rows(len(items))
+            if allowed < len(items):
+                charging_manager._note_refused_default_dataset_items(len(items) - allowed)
+                if allowed == 0:
+                    return ChargeResult(
+                        chargeable_within_limit=charging_manager.compute_chargeable(),
+                        charged_count=0,
+                        event_charge_limit_reached=True,
+                    )
+                items = items[:allowed]
 
         limited_count = len(items)
         if is_ppe and resolved_event:
@@ -1522,7 +1571,8 @@ async def push_data_and_charge(
                     event_charge_limit_reached=any(
                         result.event_charge_limit_reached for result in results
                     )
-                    or accepted_for_items < len(limited_items),
+                    or accepted_for_items < len(limited_items)
+                    or unpaid_past_limit,
                 )
 
 
@@ -1531,10 +1581,170 @@ async def push_data_and_charge(
         return ChargeResult(
             chargeable_within_limit=charging_manager.compute_chargeable(),
             charged_count=0,
-            event_charge_limit_reached=False,
+            event_charge_limit_reached=unpaid_past_limit,
         )
 
 
+def _validate_row_charges(charges: Any) -> Dict[str, int]:
+    if not isinstance(charges, dict) or not charges:
+        raise ValueError('charges must be a non-empty dict of {event_name: count_per_row}')
+    out: Dict[str, int] = {}
+    for name, count in charges.items():
+        if not isinstance(name, str) or not name:
+            raise ValueError('charges keys must be event names')
+        if is_default_dataset_item_event(name):
+            raise ValueError(f"Cannot charge for synthetic event '{name}' manually")
+        if not isinstance(count, int) or isinstance(count, bool) or count < 1:
+            raise ValueError(f'charges[{name!r}] must be a positive integer (events per row)')
+        out[name] = count
+    return out
+
+
+async def push_data_with_charges(
+    charging_manager: ChargingManager,
+    items: Union[Dict[str, Any], List[Dict[str, Any]]],
+    charges: Dict[str, int],
+    is_default_dataset: bool,
+    push_fn: Any,
+) -> ChargeResult:
+    """Push rows that each cost several events, atomically per row.
+
+    ``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
+    A row is stored only when every one of its events was charged: the full set
+    is charged first, rows the budget cannot fully pay for are not stored, and
+    any operation that came back short is voided and re-charged for the rows
+    that are kept, so no row is ever stored partly paid and no event is billed
+    for a row that was not stored. Unpriced events are ignored. On the default
+    dataset the default-dataset-item event (when priced) is added per row.
+    """
+    charges = _validate_row_charges(charges)
+    rows_in = items if isinstance(items, list) else [items]
+    if not rows_in:
+        return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
+
+    async with charging_manager._operation_lock:
+        if charging_manager._closed:
+            raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
+        voids_confirmed = await charging_manager._retry_pending_voids()
+
+        if charging_manager._pricing_info is None:
+            await _gated_push(push_fn, rows_in)
+            return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
+
+        per_row: Dict[str, int] = {
+            name: n for name, n in charges.items()
+            if (charging_manager._calculate_event_price(name) or Decimal('0')) > 0
+        }
+        if is_default_dataset:
+            default_event = charging_manager._config.get('default_dataset_item_event_name') or \
+                resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
+            if default_event and default_event not in per_row and \
+                    (charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
+                per_row[default_event] = 1
+        if not per_row:
+            # Every listed event is free: a plain (unpaid) push.
+            return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
+        if not voids_confirmed and any(not is_default_dataset_item_event(n) for n in per_row):
+            raise RuntimeError('An earlier charge for a failed dataset push could not be voided yet; retry the push')
+
+        # Rows the budget can pay for in full (no one-event overdraw: a row is all or nothing).
+        row_cost = sum(
+            (charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
+        )
+        rows = len(rows_in)
+        if charging_manager._max_total_charge_usd is not None and row_cost > 0:
+            remaining = charging_manager._max_total_charge_usd - charging_manager._total_charged_usd
+            rows = min(rows, max(0, int(remaining / row_cost)))
+
+        ops: List[tuple] = []  # (event, charged, op_id, uncertain)
+
+        async def undo(entries: List[tuple]) -> None:
+            for name, charged, op_id, uncertain in entries:
+                if charged <= 0:
+                    continue
+                if op_id is None:
+                    charging_manager._unwind_reservation(name, charged)
+                elif uncertain:
+                    await charging_manager._cancel_uncertain_charge(name, op_id, charged)
+                else:
+                    await charging_manager._void_charge(name, op_id, charged)
+
+        for _attempt in range(4):
+            if rows <= 0:
+                break
+            ops = []
+            short_to: Optional[int] = None
+            for name, n in sorted(per_row.items(), key=lambda kv: (is_default_dataset_item_event(kv[0]), kv[0])):
+                want = n * rows
+                charging_manager._take_last_accepted_operation_id()
+                try:
+                    r = await charging_manager._charge_exclusive(name, want)
+                    op_id = charging_manager._take_last_accepted_operation_id()
+                    ops.append((name, r.charged_count, op_id, False))
+                    got = r.charged_count
+                except ChargeUncertainError as unconfirmed:
+                    # Not confirmed: the intent stays pending and is re-posted at exit,
+                    # so it counts as accepted (items stored, billed once).
+                    ops.append((name, unconfirmed.count, unconfirmed.operation_id, True))
+                    got = unconfirmed.count
+                if got < want:
+                    short_to = got // n
+                    break
+            if short_to is None:
+                break
+            # Some event came back short: void this attempt in full and retry with
+            # the rows that event can pay for (a void restores the budget).
+            await undo(ops)
+            ops = []
+            rows = short_to  # got < n * rows, so short_to < rows
+
+        if rows <= 0 or not ops:
+            await undo(ops)
+            charging_manager._mark_charge_limit_hit()
+            if is_default_dataset:
+                charging_manager._note_refused_default_dataset_items(len(rows_in))
+            return ChargeResult(
+                chargeable_within_limit=charging_manager.compute_chargeable(),
+                charged_count=0,
+                event_charge_limit_reached=True,
+            )
+
+        kept = rows_in[:rows]
+        if rows < len(rows_in):
+            charging_manager._mark_charge_limit_hit()
+            if is_default_dataset:
+                charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
+        try:
+            await _gated_push(push_fn, kept)
+        except Exception as push_err:
+            logger.error('Dataset push failed after charges accepted (rows=%s): %s; voiding the charges', rows, push_err)
+            if _push_certainly_failed(push_err):
+                await undo(ops)
+            raise
+        return ChargeResult(
+            chargeable_within_limit=charging_manager.compute_chargeable(),
+            charged_count=rows,
+            event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
+        )
+
+
+async def _unpaid_push_locked(
+    charging_manager: ChargingManager, rows: List[Dict[str, Any]], is_default_dataset: bool, push_fn: Any
+) -> ChargeResult:
+    """A push whose rows carry no priced event (caller holds the operation lock)."""
+    past_limit = charging_manager.is_charge_limit_reached()
+    allowed = charging_manager._take_unpaid_rows(len(rows)) if is_default_dataset else len(rows)
+    if allowed < len(rows):
+        charging_manager._note_refused_default_dataset_items(len(rows) - allowed)
+    if allowed > 0:
+        await _gated_push(push_fn, rows[:allowed])
+    return ChargeResult(
+        chargeable_within_limit=charging_manager.compute_chargeable(),
+        charged_count=0,
+        event_charge_limit_reached=past_limit,
+    )
+
+
 def _push_is_paid_per_item(
     charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
 ) -> bool:

{scrapely_python-0.1.16.dist-info → scrapely_python-0.1.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.16
+Version: 0.1.18
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.16.dist-info → scrapely_python-0.1.18.dist-info}/RECORD

@@ -1,9 +1,9 @@
-scrapely/__init__.py,sha256=liCCoe7VolXKiiQlJmaBGJt7DlkLFi904rNpFMT-kX8,5746
+scrapely/__init__.py,sha256=EHGVITjobhdZu3nhkmBmECrlftUMCsnDyLAIxdnZ_7A,5746
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
-scrapely/actor.py,sha256=1MXGyHql5sRfh15HMRY8vAp_TNr_yhavDALlHBcjLO0,80810
+scrapely/actor.py,sha256=Lsb4_14RCGQy-EWACnqijo7Hz5xvPKSDZcu4B8_ZrHc,81685
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
 scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
-scrapely/charging.py,sha256=HQHDkJ9buhlgArmmAC0I4lON8v-B6jgLD594-MpB-Rs,68905
+scrapely/charging.py,sha256=maRgijO8vM8mCb_S9lnvF99zNR_G8sB3LsubU1r6t6M,79326
 scrapely/charging_client.py,sha256=-4q-Ww7gu9EZXpM9QIn1Vd4vwAXmgL_WEKqaQIx3ViA,3961
 scrapely/configuration.py,sha256=Of-lKm7hoPGUQO8d08kV7fAy2co9QTfb8Of_8ds5PSA,17028
 scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
@@ -39,10 +39,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
 scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
 scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
 scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
-scrapely_python-0.1.16.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
-scrapely_python-0.1.16.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
-scrapely_python-0.1.16.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
-scrapely_python-0.1.16.dist-info/METADATA,sha256=wxPrakVU3_4JfSWaOgNs6hYKp3vkCbssBHR5OH9Jcoo,3487
-scrapely_python-0.1.16.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-scrapely_python-0.1.16.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.16.dist-info/RECORD,,
+scrapely_python-0.1.18.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
+scrapely_python-0.1.18.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
+scrapely_python-0.1.18.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
+scrapely_python-0.1.18.dist-info/METADATA,sha256=3znh1pGrKvP0jNRAftMgiGSKuBG1c3g8UWJtgMYI95g,3487
+scrapely_python-0.1.18.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.18.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.18.dist-info/RECORD,,