scrapely-python 0.1.17__py3-none-any.whl → 0.1.18__py3-none-any.whl

scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.17'
+__version__ = '0.1.18'
 __all__ = [
     # Core
     'Actor',

scrapely/actor.py

@@ -28,6 +28,7 @@ from .charging import (
     ChargingManager,
     is_default_dataset_item_event,
     push_data_and_charge,
+    push_data_with_charges,
     set_active_charging_manager,
 )
 from .configuration import Configuration
@@ -781,6 +782,7 @@ class Actor(metaclass=_ActorMeta):
         charged_event_name: Optional[str] = None,
         *,
         event_name: Optional[str] = None,
+        charges: Optional[Dict[str, int]] = None,
     ) -> ChargeResult:
         """Push data to the default dataset, charging pay-per-event items.
 
@@ -796,6 +798,10 @@ class Actor(metaclass=_ActorMeta):
                 (Apify Python SDK name; also accepted positionally).
             event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
                 kept so existing callers keep working.
+            charges: Several events per row, charged atomically: ``{'result': 1, 'page': 3}``
+                charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
+                only when all of its events were charged; rows the spending limit cannot
+                pay for in full are not stored. Cannot be combined with ``charged_event_name``.
 
         Returns:
             ChargeResult describing how many items were charged and pushed.
@@ -818,6 +824,16 @@ class Actor(metaclass=_ActorMeta):
 
         items = data if isinstance(data, list) else [data]
         charging_manager = cls.get_charging_manager()
+        if charges is not None:
+            if event_name is not None:
+                raise ValueError('Pass either charged_event_name or charges, not both')
+            return await push_data_with_charges(
+                charging_manager,
+                items,
+                charges,
+                True,  # Actor.push_data always targets the default dataset
+                cls._default_dataset.push_data,
+            )
         return await push_data_and_charge(
             charging_manager,
             items,

scrapely/charging.py

@@ -1585,6 +1585,166 @@ async def push_data_and_charge(
         )
 
 
+def _validate_row_charges(charges: Any) -> Dict[str, int]:
+    if not isinstance(charges, dict) or not charges:
+        raise ValueError('charges must be a non-empty dict of {event_name: count_per_row}')
+    out: Dict[str, int] = {}
+    for name, count in charges.items():
+        if not isinstance(name, str) or not name:
+            raise ValueError('charges keys must be event names')
+        if is_default_dataset_item_event(name):
+            raise ValueError(f"Cannot charge for synthetic event '{name}' manually")
+        if not isinstance(count, int) or isinstance(count, bool) or count < 1:
+            raise ValueError(f'charges[{name!r}] must be a positive integer (events per row)')
+        out[name] = count
+    return out
+
+
+async def push_data_with_charges(
+    charging_manager: ChargingManager,
+    items: Union[Dict[str, Any], List[Dict[str, Any]]],
+    charges: Dict[str, int],
+    is_default_dataset: bool,
+    push_fn: Any,
+) -> ChargeResult:
+    """Push rows that each cost several events, atomically per row.
+
+    ``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
+    A row is stored only when every one of its events was charged: the full set
+    is charged first, rows the budget cannot fully pay for are not stored, and
+    any operation that came back short is voided and re-charged for the rows
+    that are kept, so no row is ever stored partly paid and no event is billed
+    for a row that was not stored. Unpriced events are ignored. On the default
+    dataset the default-dataset-item event (when priced) is added per row.
+    """
+    charges = _validate_row_charges(charges)
+    rows_in = items if isinstance(items, list) else [items]
+    if not rows_in:
+        return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
+
+    async with charging_manager._operation_lock:
+        if charging_manager._closed:
+            raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
+        voids_confirmed = await charging_manager._retry_pending_voids()
+
+        if charging_manager._pricing_info is None:
+            await _gated_push(push_fn, rows_in)
+            return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
+
+        per_row: Dict[str, int] = {
+            name: n for name, n in charges.items()
+            if (charging_manager._calculate_event_price(name) or Decimal('0')) > 0
+        }
+        if is_default_dataset:
+            default_event = charging_manager._config.get('default_dataset_item_event_name') or \
+                resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
+            if default_event and default_event not in per_row and \
+                    (charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
+                per_row[default_event] = 1
+        if not per_row:
+            # Every listed event is free: a plain (unpaid) push.
+            return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
+        if not voids_confirmed and any(not is_default_dataset_item_event(n) for n in per_row):
+            raise RuntimeError('An earlier charge for a failed dataset push could not be voided yet; retry the push')
+
+        # Rows the budget can pay for in full (no one-event overdraw: a row is all or nothing).
+        row_cost = sum(
+            (charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
+        )
+        rows = len(rows_in)
+        if charging_manager._max_total_charge_usd is not None and row_cost > 0:
+            remaining = charging_manager._max_total_charge_usd - charging_manager._total_charged_usd
+            rows = min(rows, max(0, int(remaining / row_cost)))
+
+        ops: List[tuple] = []  # (event, charged, op_id, uncertain)
+
+        async def undo(entries: List[tuple]) -> None:
+            for name, charged, op_id, uncertain in entries:
+                if charged <= 0:
+                    continue
+                if op_id is None:
+                    charging_manager._unwind_reservation(name, charged)
+                elif uncertain:
+                    await charging_manager._cancel_uncertain_charge(name, op_id, charged)
+                else:
+                    await charging_manager._void_charge(name, op_id, charged)
+
+        for _attempt in range(4):
+            if rows <= 0:
+                break
+            ops = []
+            short_to: Optional[int] = None
+            for name, n in sorted(per_row.items(), key=lambda kv: (is_default_dataset_item_event(kv[0]), kv[0])):
+                want = n * rows
+                charging_manager._take_last_accepted_operation_id()
+                try:
+                    r = await charging_manager._charge_exclusive(name, want)
+                    op_id = charging_manager._take_last_accepted_operation_id()
+                    ops.append((name, r.charged_count, op_id, False))
+                    got = r.charged_count
+                except ChargeUncertainError as unconfirmed:
+                    # Not confirmed: the intent stays pending and is re-posted at exit,
+                    # so it counts as accepted (items stored, billed once).
+                    ops.append((name, unconfirmed.count, unconfirmed.operation_id, True))
+                    got = unconfirmed.count
+                if got < want:
+                    short_to = got // n
+                    break
+            if short_to is None:
+                break
+            # Some event came back short: void this attempt in full and retry with
+            # the rows that event can pay for (a void restores the budget).
+            await undo(ops)
+            ops = []
+            rows = short_to  # got < n * rows, so short_to < rows
+
+        if rows <= 0 or not ops:
+            await undo(ops)
+            charging_manager._mark_charge_limit_hit()
+            if is_default_dataset:
+                charging_manager._note_refused_default_dataset_items(len(rows_in))
+            return ChargeResult(
+                chargeable_within_limit=charging_manager.compute_chargeable(),
+                charged_count=0,
+                event_charge_limit_reached=True,
+            )
+
+        kept = rows_in[:rows]
+        if rows < len(rows_in):
+            charging_manager._mark_charge_limit_hit()
+            if is_default_dataset:
+                charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
+        try:
+            await _gated_push(push_fn, kept)
+        except Exception as push_err:
+            logger.error('Dataset push failed after charges accepted (rows=%s): %s; voiding the charges', rows, push_err)
+            if _push_certainly_failed(push_err):
+                await undo(ops)
+            raise
+        return ChargeResult(
+            chargeable_within_limit=charging_manager.compute_chargeable(),
+            charged_count=rows,
+            event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
+        )
+
+
+async def _unpaid_push_locked(
+    charging_manager: ChargingManager, rows: List[Dict[str, Any]], is_default_dataset: bool, push_fn: Any
+) -> ChargeResult:
+    """A push whose rows carry no priced event (caller holds the operation lock)."""
+    past_limit = charging_manager.is_charge_limit_reached()
+    allowed = charging_manager._take_unpaid_rows(len(rows)) if is_default_dataset else len(rows)
+    if allowed < len(rows):
+        charging_manager._note_refused_default_dataset_items(len(rows) - allowed)
+    if allowed > 0:
+        await _gated_push(push_fn, rows[:allowed])
+    return ChargeResult(
+        chargeable_within_limit=charging_manager.compute_chargeable(),
+        charged_count=0,
+        event_charge_limit_reached=past_limit,
+    )
+
+
 def _push_is_paid_per_item(
     charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
 ) -> bool:

{scrapely_python-0.1.17.dist-info → scrapely_python-0.1.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.17
+Version: 0.1.18
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.17.dist-info → scrapely_python-0.1.18.dist-info}/RECORD

@@ -1,9 +1,9 @@
-scrapely/__init__.py,sha256=TqM8pFfiE1_j1M69_2BOtRVAy1jMK6HzvKNpmYtNnuo,5746
+scrapely/__init__.py,sha256=EHGVITjobhdZu3nhkmBmECrlftUMCsnDyLAIxdnZ_7A,5746
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
-scrapely/actor.py,sha256=1MXGyHql5sRfh15HMRY8vAp_TNr_yhavDALlHBcjLO0,80810
+scrapely/actor.py,sha256=Lsb4_14RCGQy-EWACnqijo7Hz5xvPKSDZcu4B8_ZrHc,81685
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
 scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
-scrapely/charging.py,sha256=ImnjF4RfzLbpRp3JZIl2RM-TfHp22p0Xzt0NSVs2M1c,71477
+scrapely/charging.py,sha256=maRgijO8vM8mCb_S9lnvF99zNR_G8sB3LsubU1r6t6M,79326
 scrapely/charging_client.py,sha256=-4q-Ww7gu9EZXpM9QIn1Vd4vwAXmgL_WEKqaQIx3ViA,3961
 scrapely/configuration.py,sha256=Of-lKm7hoPGUQO8d08kV7fAy2co9QTfb8Of_8ds5PSA,17028
 scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
@@ -39,10 +39,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
 scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
 scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
 scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
-scrapely_python-0.1.17.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
-scrapely_python-0.1.17.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
-scrapely_python-0.1.17.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
-scrapely_python-0.1.17.dist-info/METADATA,sha256=CUcmUdVmdGoDotCHQvZUa2eYIYKugeJZvsbA9tWrVRg,3487
-scrapely_python-0.1.17.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-scrapely_python-0.1.17.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.17.dist-info/RECORD,,
+scrapely_python-0.1.18.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
+scrapely_python-0.1.18.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
+scrapely_python-0.1.18.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
+scrapely_python-0.1.18.dist-info/METADATA,sha256=3znh1pGrKvP0jNRAftMgiGSKuBG1c3g8UWJtgMYI95g,3487
+scrapely_python-0.1.18.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.18.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.18.dist-info/RECORD,,