scrapely-python 0.1.20__py3-none-any.whl → 0.1.22__py3-none-any.whl

scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.20'
+__version__ = '0.1.22'
 __all__ = [
     # Core
     'Actor',

scrapely/actor.py

@@ -783,6 +783,8 @@ class Actor(metaclass=_ActorMeta):
         *,
         event_name: Optional[str] = None,
         charges: Optional[Dict[str, int]] = None,
+        charge_per: str = 'row',
+        dataset: Optional[Union[str, Dataset]] = None,
     ) -> ChargeResult:
         """Push data to the default dataset, charging pay-per-event items.
 
@@ -802,6 +804,16 @@ class Actor(metaclass=_ActorMeta):
                 charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
                 only when all of its events were charged; rows the spending limit cannot
                 pay for in full are not stored. Cannot be combined with ``charged_event_name``.
+            charge_per: ``'row'`` (default) charges ``charges`` for every row; ``'batch'``
+                charges them once for the whole push, e.g.
+                ``push_data(page_rows, charges={'document': 1}, charge_per='batch')``:
+                all rows are stored if the charge is accepted, none if it is refused.
+            dataset: Push to this named dataset (name, or a Dataset from ``open_dataset``)
+                instead of the default one, with the same charge-first semantics:
+                ``push_data(rows, charged_event_name='comment', dataset=f'comments-{run_id}')``.
+                Rows of a named dataset carry no implicit default-dataset-item event and are
+                not gated by the default dataset's spending-limit safety net; charged pushes
+                store only the rows that were charged.
 
         Returns:
             ChargeResult describing how many items were charged and pushed.
@@ -822,6 +834,13 @@ class Actor(metaclass=_ActorMeta):
         if cls._default_dataset is None:
             cls._default_dataset = await cls.open_dataset()
 
+        target = cls._default_dataset
+        if dataset is not None:
+            target = dataset if isinstance(dataset, Dataset) else await cls.open_dataset(name=dataset)
+        is_default = target is cls._default_dataset or (
+            getattr(target, 'id', None) is not None and target.id == getattr(cls._default_dataset, 'id', None)
+        )
+
         items = data if isinstance(data, list) else [data]
         charging_manager = cls.get_charging_manager()
         if charges is not None:
@@ -831,15 +850,18 @@ class Actor(metaclass=_ActorMeta):
                 charging_manager,
                 items,
                 charges,
-                True,  # Actor.push_data always targets the default dataset
-                cls._default_dataset.push_data,
+                is_default,
+                target.push_data,
+                charge_per=charge_per,
             )
+        if charge_per != 'row':
+            raise ValueError("charge_per needs charges={...}")
         return await push_data_and_charge(
             charging_manager,
             items,
             event_name,
-            True,  # Actor.push_data always targets the default dataset
-            cls._default_dataset.push_data,
+            is_default,
+            target.push_data,
         )
 
     @classmethod

scrapely/charging.py

@@ -1657,9 +1657,15 @@ async def push_data_with_charges(
     charges: Dict[str, int],
     is_default_dataset: bool,
     push_fn: Any,
+    charge_per: str = 'row',
 ) -> ChargeResult:
     """Push rows that each cost several events, atomically per row.
 
+    ``charge_per='batch'``: ``charges`` are charged ONCE for the whole push (e.g. one
+    ``document`` for all of its page rows). If every event is accepted all rows are
+    stored, otherwise none (and nothing stays billed). The prepaid-row balance is
+    neither used nor changed.
+
     ``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
     A row is stored only when every one of its events was charged: the full set
     is charged first, rows the budget cannot fully pay for are not stored, and
@@ -1669,6 +1675,9 @@ async def push_data_with_charges(
     dataset the default-dataset-item event (when priced) is added per row.
     """
     charges = _validate_row_charges(charges)
+    if charge_per not in ('row', 'batch'):
+        raise ValueError("charge_per must be 'row' or 'batch'")
+    batch = charge_per == 'batch'
     rows_in = items if isinstance(items, list) else [items]
     if not rows_in:
         return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
@@ -1691,7 +1700,8 @@ async def push_data_with_charges(
                 resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
             if default_event and default_event not in per_row and \
                     (charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
-                per_row[default_event] = 1
+                # Per stored row, also for a batch.
+                per_row[default_event] = len(rows_in) if batch else 1
         if not per_row:
             # Every listed event is free: a plain (unpaid) push.
             return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
@@ -1702,7 +1712,8 @@ async def push_data_with_charges(
         row_cost = sum(
             (charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
         )
-        rows = len(rows_in)
+        # Units charged for: rows, or the one batch (row_cost is then the batch's cost).
+        rows = 1 if batch else len(rows_in)
         if charging_manager._max_total_charge_usd is not None and row_cost > 0:
             remaining = charging_manager._remaining_budget_usd()
             rows = min(rows, max(0, int(remaining / row_cost)))
@@ -1775,11 +1786,11 @@ async def push_data_with_charges(
                 event_charge_limit_reached=True,
             )
 
-        kept = rows_in[:rows]
-        if rows < len(rows_in):
+        kept = rows_in if batch else rows_in[:rows]
+        if len(kept) < len(rows_in):
             charging_manager._mark_charge_limit_hit()
             if is_default_dataset:
-                charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
+                charging_manager._note_refused_default_dataset_items(len(rows_in) - len(kept))
         try:
             await _gated_push(push_fn, kept)
         except Exception as push_err:
@@ -1789,8 +1800,8 @@ async def push_data_with_charges(
             raise
         return ChargeResult(
             chargeable_within_limit=charging_manager.compute_chargeable(),
-            charged_count=rows,
-            event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
+            charged_count=len(kept),
+            event_charge_limit_reached=len(kept) < len(rows_in) or charging_manager.is_charge_limit_reached(),
         )
 
 

{scrapely_python-0.1.20.dist-info → scrapely_python-0.1.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.20
+Version: 0.1.22
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.20.dist-info → scrapely_python-0.1.22.dist-info}/RECORD

@@ -1,9 +1,9 @@
-scrapely/__init__.py,sha256=i4A_AaMoKY5cMXqvx7Df9eyucWihcJTRz4ra4HobviI,5746
+scrapely/__init__.py,sha256=cxlsfPD_FlaCCerWYImS5rGRnKxE9INU61hDl3C04Y0,5746
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
-scrapely/actor.py,sha256=Lsb4_14RCGQy-EWACnqijo7Hz5xvPKSDZcu4B8_ZrHc,81685
+scrapely/actor.py,sha256=yLy05XzoFwslw0D4aSM38rDIGd81JvPqpngY467hWxQ,82969
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
 scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
-scrapely/charging.py,sha256=uba7lPFx7ENY4aPjqQAN-SDY1wjXZ83jLxcnyGxYDTE,83121
+scrapely/charging.py,sha256=ns-kuUJtl8hBI4-3iBisC62BggJG67yYOa1yVVItoUI,83800
 scrapely/charging_client.py,sha256=-4q-Ww7gu9EZXpM9QIn1Vd4vwAXmgL_WEKqaQIx3ViA,3961
 scrapely/configuration.py,sha256=Of-lKm7hoPGUQO8d08kV7fAy2co9QTfb8Of_8ds5PSA,17028
 scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
@@ -39,10 +39,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
 scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
 scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
 scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
-scrapely_python-0.1.20.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
-scrapely_python-0.1.20.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
-scrapely_python-0.1.20.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
-scrapely_python-0.1.20.dist-info/METADATA,sha256=ilbOda4UXoNvHh1O91s-GUv8aQ9C8gA1HF_48yb2Tp8,3487
-scrapely_python-0.1.20.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-scrapely_python-0.1.20.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.20.dist-info/RECORD,,
+scrapely_python-0.1.22.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
+scrapely_python-0.1.22.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
+scrapely_python-0.1.22.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
+scrapely_python-0.1.22.dist-info/METADATA,sha256=K-JZZELeDs7kIXmXodCXXVF3YCvozeqESmkQ5StszVw,3487
+scrapely_python-0.1.22.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.22.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.22.dist-info/RECORD,,