scrapely-python 0.1.17__tar.gz → 0.1.19__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. {scrapely_python-0.1.17/src/scrapely_python.egg-info → scrapely_python-0.1.19}/PKG-INFO +1 -1
  2. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/pyproject.toml +1 -1
  3. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/__init__.py +1 -1
  4. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/actor.py +16 -0
  5. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/charging.py +213 -5
  6. {scrapely_python-0.1.17 → scrapely_python-0.1.19/src/scrapely_python.egg-info}/PKG-INFO +1 -1
  7. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_charge_limit_guard.py +185 -0
  8. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/LICENSE +0 -0
  9. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/LICENSE-APACHE +0 -0
  10. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/NOTICE +0 -0
  11. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/README.md +0 -0
  12. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/setup.cfg +0 -0
  13. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/_browser_locations.py +0 -0
  14. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/browser_configuration.py +0 -0
  15. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/cdp_retry.py +0 -0
  16. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/charging_client.py +0 -0
  17. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/configuration.py +0 -0
  18. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/crypto/__init__.py +0 -0
  19. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/crypto/_secrets.py +0 -0
  20. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/errors.py +0 -0
  21. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/events/__init__.py +0 -0
  22. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/events/_types.py +0 -0
  23. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/events/event_manager.py +0 -0
  24. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/log.py +0 -0
  25. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/patch_browser_pool.py +0 -0
  26. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/pricing_types.py +0 -0
  27. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/proxy_configuration.py +0 -0
  28. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/py.typed +0 -0
  29. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/request_loaders/__init__.py +0 -0
  30. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/request_loaders/_scrapely_request_list.py +0 -0
  31. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/solver.py +0 -0
  32. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/__init__.py +0 -0
  33. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/_file_system/__init__.py +0 -0
  34. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/_file_system/_key_value_store_client.py +0 -0
  35. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/_file_system/_storage_client.py +0 -0
  36. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/_metadata.py +0 -0
  37. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/api_client.py +0 -0
  38. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/client.py +0 -0
  39. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/patch_storage.py +0 -0
  40. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_dataset_client.py +0 -0
  41. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_kvs_client.py +0 -0
  42. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_rq_client.py +0 -0
  43. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_storage_client.py +0 -0
  44. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/smart_scrapely_storage_client.py +0 -0
  45. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/types.py +0 -0
  46. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/utils.py +0 -0
  47. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage_clients/__init__.py +0 -0
  48. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storages/__init__.py +0 -0
  49. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/types.py +0 -0
  50. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/webhook.py +0 -0
  51. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/SOURCES.txt +0 -0
  52. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/dependency_links.txt +0 -0
  53. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/requires.txt +0 -0
  54. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/top_level.txt +0 -0
  55. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_api_client_retries.py +0 -0
  56. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_apify_parity.py +0 -0
  57. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_browser_configuration.py +0 -0
  58. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_cdp_retry.py +0 -0
  59. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_charge_retries.py +0 -0
  60. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_charging_parity.py +0 -0
  61. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_charging_pending.py +0 -0
  62. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_input_proxy_aliases.py +0 -0
  63. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_parity.py +0 -0
  64. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_patch_browser_pool.py +0 -0
  65. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_proxy_configuration.py +0 -0
  66. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_push_idempotency.py +0 -0
  67. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_run_control.py +0 -0
  68. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_secrets.py +0 -0
  69. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_solver.py +0 -0
  70. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_storage.py +0 -0
  71. {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_terminal_status_message.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-python
3
- Version: 0.1.17
3
+ Version: 0.1.19
4
4
  Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
5
5
  Author-email: Scrapely <support@scrape.ly>
6
6
  License-Expression: MIT AND Apache-2.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "scrapely-python"
7
- version = "0.1.17"
7
+ version = "0.1.19"
8
8
  description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
9
9
  readme = "README.md"
10
10
  license = "MIT AND Apache-2.0"
@@ -132,7 +132,7 @@ from .errors import (
132
132
  raise_for_status,
133
133
  )
134
134
 
135
- __version__ = '0.1.17'
135
+ __version__ = '0.1.19'
136
136
  __all__ = [
137
137
  # Core
138
138
  'Actor',
@@ -28,6 +28,7 @@ from .charging import (
28
28
  ChargingManager,
29
29
  is_default_dataset_item_event,
30
30
  push_data_and_charge,
31
+ push_data_with_charges,
31
32
  set_active_charging_manager,
32
33
  )
33
34
  from .configuration import Configuration
@@ -781,6 +782,7 @@ class Actor(metaclass=_ActorMeta):
781
782
  charged_event_name: Optional[str] = None,
782
783
  *,
783
784
  event_name: Optional[str] = None,
785
+ charges: Optional[Dict[str, int]] = None,
784
786
  ) -> ChargeResult:
785
787
  """Push data to the default dataset, charging pay-per-event items.
786
788
 
@@ -796,6 +798,10 @@ class Actor(metaclass=_ActorMeta):
796
798
  (Apify Python SDK name; also accepted positionally).
797
799
  event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
798
800
  kept so existing callers keep working.
801
+ charges: Several events per row, charged atomically: ``{'result': 1, 'page': 3}``
802
+ charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
803
+ only when all of its events were charged; rows the spending limit cannot
804
+ pay for in full are not stored. Cannot be combined with ``charged_event_name``.
799
805
 
800
806
  Returns:
801
807
  ChargeResult describing how many items were charged and pushed.
@@ -818,6 +824,16 @@ class Actor(metaclass=_ActorMeta):
818
824
 
819
825
  items = data if isinstance(data, list) else [data]
820
826
  charging_manager = cls.get_charging_manager()
827
+ if charges is not None:
828
+ if event_name is not None:
829
+ raise ValueError('Pass either charged_event_name or charges, not both')
830
+ return await push_data_with_charges(
831
+ charging_manager,
832
+ items,
833
+ charges,
834
+ True, # Actor.push_data always targets the default dataset
835
+ cls._default_dataset.push_data,
836
+ )
821
837
  return await push_data_and_charge(
822
838
  charging_manager,
823
839
  items,
@@ -240,6 +240,12 @@ class ChargingManager:
240
240
  self._charged_events: Dict[str, int] = {}
241
241
  self._max_total_charge_usd: Optional[Decimal] = None
242
242
  self._total_charged_usd: Decimal = Decimal('0.0')
243
+ # Run spend the platform counts against maxTotalChargeUsd that this SDK
244
+ # does not charge itself: compute/proxy/browser usage and server-billed
245
+ # events. Learnt from the charge endpoint's remainingBudgetNd (only ever
246
+ # grows), so the local budget gates on the run's true remaining budget.
247
+ self._platform_other_usd: Decimal = Decimal('0')
248
+ self._last_remaining_budget_nd: Optional[int] = None
243
249
  self._charge_lock = RLock()
244
250
  self._flush_lock = asyncio.Lock()
245
251
  # Serializes budget calculation/reservation with PPE-aware dataset pushes.
@@ -288,6 +294,27 @@ class ChargingManager:
288
294
  self._refused_default_dataset_items,
289
295
  )
290
296
 
297
+ def _remaining_budget_usd(self) -> Decimal:
298
+ """maxTotalChargeUsd minus this SDK's charges and the platform's other run spend."""
299
+ assert self._max_total_charge_usd is not None
300
+ return self._max_total_charge_usd - self._total_charged_usd - self._platform_other_usd
301
+
302
+ def _sync_platform_budget(self) -> None:
303
+ """Fold the last charge response's remainingBudgetNd into the local budget.
304
+
305
+ Called after the local reservation matches what the platform accepted.
306
+ The platform's remaining budget = max - (our charges + usage + server
307
+ events), so ``other`` = max - remaining - our charges. Monotonic: usage
308
+ never shrinks, and a stale answer must not hand budget back.
309
+ """
310
+ nd = self._last_remaining_budget_nd
311
+ self._last_remaining_budget_nd = None
312
+ if nd is None or self._max_total_charge_usd is None or nd < 0:
313
+ return
314
+ other = self._max_total_charge_usd - Decimal(nd) / Decimal(1_000_000_000) - self._total_charged_usd
315
+ if other > self._platform_other_usd:
316
+ self._platform_other_usd = other
317
+
291
318
  def is_charge_limit_reached(self) -> bool:
292
319
  """Whether this pay-per-event run has used up its spending limit.
293
320
 
@@ -307,7 +334,7 @@ class ChargingManager:
307
334
  prices = [price for price in self._per_event_prices.values() if price > 0]
308
335
  if not prices:
309
336
  return False
310
- return self._max_total_charge_usd - self._total_charged_usd < min(prices)
337
+ return self._remaining_budget_usd() < min(prices)
311
338
 
312
339
  def _mark_charge_limit_hit(self) -> None:
313
340
  if self._pricing_info is not None:
@@ -795,6 +822,7 @@ class ChargingManager:
795
822
  idempotency_key=idempotency_key,
796
823
  deadline=deadline,
797
824
  )
825
+ self._last_remaining_budget_nd = _read_remaining_budget_nd(body)
798
826
  return _read_accepted_count(body, count)
799
827
  except Exception as e:
800
828
  last_error = e
@@ -957,7 +985,7 @@ class ChargingManager:
957
985
  # legacy one-event overdraw while the budget is not yet exceeded.
958
986
  chargeable_count = count
959
987
  if self._max_total_charge_usd is not None and affordability_price > 0:
960
- remaining = self._max_total_charge_usd - self._total_charged_usd
988
+ remaining = self._remaining_budget_usd()
961
989
  max_count_by_limit = max(0, int(remaining / affordability_price))
962
990
  if count > max_count_by_limit:
963
991
  chargeable_count = (
@@ -1045,6 +1073,15 @@ class ChargingManager:
1045
1073
  charged_count=chargeable_count,
1046
1074
  event_charge_limit_reached=self._is_event_limit_reached(event_name),
1047
1075
  )
1076
+ self._sync_platform_budget()
1077
+ if self._max_total_charge_usd is not None and self._platform_other_usd > 0:
1078
+ # Usage may have used up what is left: report it on this result.
1079
+ result = ChargeResult(
1080
+ chargeable_within_limit=self._compute_chargeable(),
1081
+ charged_count=result.charged_count,
1082
+ event_charge_limit_reached=result.event_charge_limit_reached
1083
+ or self._is_event_limit_reached(event_name),
1084
+ )
1048
1085
  else:
1049
1086
  raise ChargeUncertainError(
1050
1087
  f'Charge for event {event_name!r} (count={chargeable_count}) '
@@ -1103,7 +1140,7 @@ class ChargingManager:
1103
1140
  if event_price is None or event_price <= 0:
1104
1141
  return None
1105
1142
 
1106
- remaining = self._max_total_charge_usd - self._total_charged_usd
1143
+ remaining = self._remaining_budget_usd()
1107
1144
  return max(0, int(remaining / event_price))
1108
1145
 
1109
1146
  def calculate_total_charged_amount(self) -> Decimal:
@@ -1165,7 +1202,7 @@ class ChargingManager:
1165
1202
  if self._max_total_charge_usd is None:
1166
1203
  return items_count
1167
1204
 
1168
- remaining = self._max_total_charge_usd - self._total_charged_usd
1205
+ remaining = self._remaining_budget_usd()
1169
1206
  max_items = int(remaining / combined_price)
1170
1207
 
1171
1208
  return min(items_count, max(0, max_items))
@@ -1226,7 +1263,7 @@ class ChargingManager:
1226
1263
  if self._max_total_charge_usd is None:
1227
1264
  return False
1228
1265
 
1229
- remaining = self._max_total_charge_usd - self._total_charged_usd
1266
+ remaining = self._remaining_budget_usd()
1230
1267
  return remaining < event_price
1231
1268
 
1232
1269
  def _compute_chargeable(self) -> Dict[str, Optional[int]]:
@@ -1264,6 +1301,17 @@ def _is_charge_limit_rejection(err: ChargeRejectedError) -> bool:
1264
1301
  )
1265
1302
 
1266
1303
 
1304
+ def _read_remaining_budget_nd(body: Any) -> Optional[int]:
1305
+ """remainingBudgetNd from a charge response (platforms that send it), else None."""
1306
+ if not isinstance(body, dict):
1307
+ return None
1308
+ data = body.get('data') if isinstance(body.get('data'), dict) else body
1309
+ raw = data.get('remainingBudgetNd') if isinstance(data, dict) else None
1310
+ if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw >= 0:
1311
+ return int(raw)
1312
+ return None
1313
+
1314
+
1267
1315
  def _read_accepted_count(body: Any, fallback: int) -> int:
1268
1316
  if not isinstance(body, dict):
1269
1317
  return fallback
@@ -1585,6 +1633,166 @@ async def push_data_and_charge(
1585
1633
  )
1586
1634
 
1587
1635
 
1636
+ def _validate_row_charges(charges: Any) -> Dict[str, int]:
1637
+ if not isinstance(charges, dict) or not charges:
1638
+ raise ValueError('charges must be a non-empty dict of {event_name: count_per_row}')
1639
+ out: Dict[str, int] = {}
1640
+ for name, count in charges.items():
1641
+ if not isinstance(name, str) or not name:
1642
+ raise ValueError('charges keys must be event names')
1643
+ if is_default_dataset_item_event(name):
1644
+ raise ValueError(f"Cannot charge for synthetic event '{name}' manually")
1645
+ if not isinstance(count, int) or isinstance(count, bool) or count < 1:
1646
+ raise ValueError(f'charges[{name!r}] must be a positive integer (events per row)')
1647
+ out[name] = count
1648
+ return out
1649
+
1650
+
1651
+ async def push_data_with_charges(
1652
+ charging_manager: ChargingManager,
1653
+ items: Union[Dict[str, Any], List[Dict[str, Any]]],
1654
+ charges: Dict[str, int],
1655
+ is_default_dataset: bool,
1656
+ push_fn: Any,
1657
+ ) -> ChargeResult:
1658
+ """Push rows that each cost several events, atomically per row.
1659
+
1660
+ ``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
1661
+ A row is stored only when every one of its events was charged: the full set
1662
+ is charged first, rows the budget cannot fully pay for are not stored, and
1663
+ any operation that came back short is voided and re-charged for the rows
1664
+ that are kept, so no row is ever stored partly paid and no event is billed
1665
+ for a row that was not stored. Unpriced events are ignored. On the default
1666
+ dataset the default-dataset-item event (when priced) is added per row.
1667
+ """
1668
+ charges = _validate_row_charges(charges)
1669
+ rows_in = items if isinstance(items, list) else [items]
1670
+ if not rows_in:
1671
+ return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
1672
+
1673
+ async with charging_manager._operation_lock:
1674
+ if charging_manager._closed:
1675
+ raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
1676
+ voids_confirmed = await charging_manager._retry_pending_voids()
1677
+
1678
+ if charging_manager._pricing_info is None:
1679
+ await _gated_push(push_fn, rows_in)
1680
+ return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
1681
+
1682
+ per_row: Dict[str, int] = {
1683
+ name: n for name, n in charges.items()
1684
+ if (charging_manager._calculate_event_price(name) or Decimal('0')) > 0
1685
+ }
1686
+ if is_default_dataset:
1687
+ default_event = charging_manager._config.get('default_dataset_item_event_name') or \
1688
+ resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
1689
+ if default_event and default_event not in per_row and \
1690
+ (charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
1691
+ per_row[default_event] = 1
1692
+ if not per_row:
1693
+ # Every listed event is free: a plain (unpaid) push.
1694
+ return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
1695
+ if not voids_confirmed and any(not is_default_dataset_item_event(n) for n in per_row):
1696
+ raise RuntimeError('An earlier charge for a failed dataset push could not be voided yet; retry the push')
1697
+
1698
+ # Rows the budget can pay for in full (no one-event overdraw: a row is all or nothing).
1699
+ row_cost = sum(
1700
+ (charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
1701
+ )
1702
+ rows = len(rows_in)
1703
+ if charging_manager._max_total_charge_usd is not None and row_cost > 0:
1704
+ remaining = charging_manager._remaining_budget_usd()
1705
+ rows = min(rows, max(0, int(remaining / row_cost)))
1706
+
1707
+ ops: List[tuple] = [] # (event, charged, op_id, uncertain)
1708
+
1709
+ async def undo(entries: List[tuple]) -> None:
1710
+ for name, charged, op_id, uncertain in entries:
1711
+ if charged <= 0:
1712
+ continue
1713
+ if op_id is None:
1714
+ charging_manager._unwind_reservation(name, charged)
1715
+ elif uncertain:
1716
+ await charging_manager._cancel_uncertain_charge(name, op_id, charged)
1717
+ else:
1718
+ await charging_manager._void_charge(name, op_id, charged)
1719
+
1720
+ for _attempt in range(4):
1721
+ if rows <= 0:
1722
+ break
1723
+ ops = []
1724
+ short_to: Optional[int] = None
1725
+ for name, n in sorted(per_row.items(), key=lambda kv: (is_default_dataset_item_event(kv[0]), kv[0])):
1726
+ want = n * rows
1727
+ charging_manager._take_last_accepted_operation_id()
1728
+ try:
1729
+ r = await charging_manager._charge_exclusive(name, want)
1730
+ op_id = charging_manager._take_last_accepted_operation_id()
1731
+ ops.append((name, r.charged_count, op_id, False))
1732
+ got = r.charged_count
1733
+ except ChargeUncertainError as unconfirmed:
1734
+ # Not confirmed: the intent stays pending and is re-posted at exit,
1735
+ # so it counts as accepted (items stored, billed once).
1736
+ ops.append((name, unconfirmed.count, unconfirmed.operation_id, True))
1737
+ got = unconfirmed.count
1738
+ if got < want:
1739
+ short_to = got // n
1740
+ break
1741
+ if short_to is None:
1742
+ break
1743
+ # Some event came back short: void this attempt in full and retry with
1744
+ # the rows that event can pay for (a void restores the budget).
1745
+ await undo(ops)
1746
+ ops = []
1747
+ rows = short_to # got < n * rows, so short_to < rows
1748
+
1749
+ if rows <= 0 or not ops:
1750
+ await undo(ops)
1751
+ charging_manager._mark_charge_limit_hit()
1752
+ if is_default_dataset:
1753
+ charging_manager._note_refused_default_dataset_items(len(rows_in))
1754
+ return ChargeResult(
1755
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1756
+ charged_count=0,
1757
+ event_charge_limit_reached=True,
1758
+ )
1759
+
1760
+ kept = rows_in[:rows]
1761
+ if rows < len(rows_in):
1762
+ charging_manager._mark_charge_limit_hit()
1763
+ if is_default_dataset:
1764
+ charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
1765
+ try:
1766
+ await _gated_push(push_fn, kept)
1767
+ except Exception as push_err:
1768
+ logger.error('Dataset push failed after charges accepted (rows=%s): %s; voiding the charges', rows, push_err)
1769
+ if _push_certainly_failed(push_err):
1770
+ await undo(ops)
1771
+ raise
1772
+ return ChargeResult(
1773
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1774
+ charged_count=rows,
1775
+ event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
1776
+ )
1777
+
1778
+
1779
+ async def _unpaid_push_locked(
1780
+ charging_manager: ChargingManager, rows: List[Dict[str, Any]], is_default_dataset: bool, push_fn: Any
1781
+ ) -> ChargeResult:
1782
+ """A push whose rows carry no priced event (caller holds the operation lock)."""
1783
+ past_limit = charging_manager.is_charge_limit_reached()
1784
+ allowed = charging_manager._take_unpaid_rows(len(rows)) if is_default_dataset else len(rows)
1785
+ if allowed < len(rows):
1786
+ charging_manager._note_refused_default_dataset_items(len(rows) - allowed)
1787
+ if allowed > 0:
1788
+ await _gated_push(push_fn, rows[:allowed])
1789
+ return ChargeResult(
1790
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1791
+ charged_count=0,
1792
+ event_charge_limit_reached=past_limit,
1793
+ )
1794
+
1795
+
1588
1796
  def _push_is_paid_per_item(
1589
1797
  charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
1590
1798
  ) -> bool:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-python
3
- Version: 0.1.17
3
+ Version: 0.1.19
4
4
  Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
5
5
  Author-email: Scrapely <support@scrape.ly>
6
6
  License-Expression: MIT AND Apache-2.0
@@ -400,3 +400,188 @@ async def test_refused_charge_402_then_plain_push_is_not_saved() -> None:
400
400
  await _plain_push(mgr, store, [{'n': n}])
401
401
 
402
402
  assert [i['n'] for i in store.items] == [0, 1]
403
+
404
+
405
+ # ---- atomic multi-event push: push_data(row, charges={...}) ----
406
+
407
+ from scrapely.charging import push_data_with_charges # noqa: E402
408
+
409
+
410
+ class MultiApi(FakeChargeApi):
411
+ """Charge endpoint priced per event, with void support (by Idempotency-Key)."""
412
+
413
+ def __init__(self, prices: Dict[str, Decimal], max_usd: Optional[Decimal]) -> None:
414
+ super().__init__(price=Decimal('0'), max_usd=max_usd)
415
+ self.prices = prices
416
+ self.ops: Dict[str, tuple] = {}
417
+ self.voids: List[str] = []
418
+
419
+ async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **_: Any) -> Dict[str, Any]:
420
+ key = (headers or {}).get('Idempotency-Key', '')
421
+ if path.endswith('/charge/void'):
422
+ name, accepted = self.ops.pop(key)
423
+ self.spent -= self.prices[name] * accepted
424
+ self.voids.append(name)
425
+ return {}
426
+ name = (json_data or {}).get('eventName')
427
+ count = int((json_data or {}).get('count', 1))
428
+ price = self.prices[name]
429
+ accepted = count
430
+ if self.max_usd is not None and price > 0:
431
+ affordable = int((self.max_usd - self.spent) / price)
432
+ if affordable < 1:
433
+ raise HttpError(402, 'charge-limit-exceeded')
434
+ accepted = min(count, affordable)
435
+ self.spent += price * accepted
436
+ self.ops[key] = (name, accepted)
437
+ return {'acceptedCount': accepted, 'requestedCount': count}
438
+
439
+
440
+ PRICES = {'result': Decimal('0.01'), 'page': Decimal('0.002')} # one row = 1 result + 3 pages = $0.016
441
+
442
+
443
+ def _multi(max_usd: str, server_max: Optional[str] = None):
444
+ api = MultiApi(PRICES, Decimal(server_max or max_usd))
445
+ return api, _manager(api, prices=dict(PRICES), max_usd=max_usd)
446
+
447
+
448
+ @pytest.mark.asyncio
449
+ async def test_multi_event_exact_budget_stores_every_paid_row() -> None:
450
+ api, mgr = _multi('0.048') # exactly 3 rows
451
+ store = Store()
452
+ r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
453
+ assert r.charged_count == 3
454
+ assert [i['n'] for i in store.items] == [0, 1, 2]
455
+ assert api.spent == Decimal('0.048')
456
+ assert mgr.is_charge_limit_reached() is True
457
+ again = await push_data_with_charges(mgr, [{'n': 9}], {'result': 1, 'page': 3}, True, store.push)
458
+ assert again.charged_count == 0 and again.event_charge_limit_reached is True
459
+ assert len(store.items) == 3 and api.spent == Decimal('0.048')
460
+
461
+
462
+ @pytest.mark.asyncio
463
+ async def test_multi_event_partial_budget_stores_only_fully_paid_rows() -> None:
464
+ api, mgr = _multi('0.040') # 2.5 rows: only 2 fully paid
465
+ store = Store()
466
+ r = await push_data_with_charges(mgr, [{'n': n} for n in range(5)], {'result': 1, 'page': 3}, True, store.push)
467
+ assert r.charged_count == 2 and r.event_charge_limit_reached is True
468
+ assert [i['n'] for i in store.items] == [0, 1]
469
+ assert api.spent == Decimal('0.032') # never a part-paid row
470
+
471
+
472
+ @pytest.mark.asyncio
473
+ async def test_multi_event_server_cap_short_voids_and_recharges() -> None:
474
+ """Local budget says 3 rows; the platform (usage ate the budget) caps 'result' at 2."""
475
+ api, mgr = _multi('1', server_max='0.032')
476
+ store = Store()
477
+ r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
478
+ # The platform's remaining budget is only learnt from the short answer, so the
479
+ # retry keeps what that answer proves affordable: fewer rows, never a part-paid one.
480
+ assert 1 <= r.charged_count <= 2
481
+ assert len(store.items) == r.charged_count
482
+ assert api.spent == Decimal('0.016') * r.charged_count
483
+ assert api.voids # the short attempt was voided, not left part-paid
484
+ assert sorted(n for n, _ in api.ops.values()) == ['page', 'result']
485
+
486
+
487
+ @pytest.mark.asyncio
488
+ async def test_multi_event_refusal_stores_nothing_and_bills_nothing() -> None:
489
+ api, mgr = _multi('0.015') # less than one full row
490
+ store = Store()
491
+ r = await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push)
492
+ assert r.charged_count == 0 and r.event_charge_limit_reached is True
493
+ assert store.items == [] and api.spent == Decimal('0')
494
+
495
+
496
+ @pytest.mark.asyncio
497
+ async def test_multi_event_push_does_not_touch_prepaid_balance() -> None:
498
+ api, mgr = _multi('0.026')
499
+ store = Store()
500
+ await mgr.charge('result', 1) # prepays one plain row
501
+ await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push) # $0.016, paid itself
502
+ assert mgr._prepaid_row_balance == 1
503
+ await _plain_push(mgr, store, [{'n': 2}]) # the prepaid row
504
+ assert [i['n'] for i in store.items] == [1, 2]
505
+
506
+
507
+ def test_multi_event_validation() -> None:
508
+ from scrapely.charging import _validate_row_charges
509
+ for bad in ({}, {'result': 0}, {'result': 1.5}, {'apify-default-dataset-item': 1}, {'': 1}, None):
510
+ with pytest.raises(ValueError):
511
+ _validate_row_charges(bad)
512
+
513
+
514
+ # ---- platform usage counts against the cap (remainingBudgetNd in charge responses) ----
515
+
516
+
517
+ class UsageApi(FakeChargeApi):
518
+ """Ledger whose cap also covers platform usage, reporting remainingBudgetNd like the server."""
519
+
520
+ def __init__(self, *, price: Decimal, max_usd: Decimal, usage: Decimal, send_remaining: bool = True) -> None:
521
+ super().__init__(price=price, max_usd=max_usd)
522
+ self.usage = usage
523
+ self.send_remaining = send_remaining
524
+
525
+ async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, **_: Any) -> Dict[str, Any]:
526
+ count = int((json_data or {}).get('count', 1))
527
+ left = self.max_usd - self.spent - self.usage
528
+ affordable = int(left / self.price)
529
+ if affordable < 1:
530
+ raise HttpError(402, 'charge-limit-exceeded')
531
+ accepted = min(count, affordable)
532
+ self.spent += self.price * accepted
533
+ body: Dict[str, Any] = {'acceptedCount': accepted, 'requestedCount': count}
534
+ if self.send_remaining:
535
+ body['remainingBudgetNd'] = int((self.max_usd - self.spent - self.usage) * 1_000_000_000)
536
+ return body
537
+
538
+
539
+ @pytest.mark.asyncio
540
+ async def test_platform_usage_shrinks_the_local_budget() -> None:
541
+ api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('0.05'), usage=Decimal('0.03'))
542
+ mgr = _manager(api, max_usd='0.05')
543
+ store = Store()
544
+
545
+ r = await mgr.charge('result', 1)
546
+ assert r.charged_count == 1
547
+ # $0.05 cap - $0.01 charged - $0.03 usage: one more event fits, not four.
548
+ assert mgr.calculate_max_event_charge_count_within_limit('result') == 1
549
+ assert mgr.is_charge_limit_reached() is False
550
+ pushed = await _charged_push(mgr, store, [{'n': n} for n in range(5)])
551
+ assert pushed.charged_count == 1 and pushed.event_charge_limit_reached is True
552
+ assert len(store.items) == 1
553
+ assert mgr.is_charge_limit_reached() is True
554
+ # The prepaid row from the first charge is still saved; nothing beyond it.
555
+ await _plain_push(mgr, store, [{'p': 1}, {'p': 2}])
556
+ assert len(store.items) == 2
557
+
558
+
559
+ @pytest.mark.asyncio
560
+ async def test_usage_learnt_from_a_charge_marks_the_limit_on_that_result() -> None:
561
+ api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('0.05'), usage=Decimal('0.04'))
562
+ mgr = _manager(api, max_usd='0.05')
563
+ r = await mgr.charge('result', 1)
564
+ assert r.charged_count == 1
565
+ assert r.event_charge_limit_reached is True
566
+ assert mgr.is_charge_limit_reached() is True
567
+
568
+
569
+ @pytest.mark.asyncio
570
+ async def test_without_remaining_budget_in_responses_behaviour_is_unchanged() -> None:
571
+ api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('0.05'), usage=Decimal('0.03'), send_remaining=False)
572
+ mgr = _manager(api, max_usd='0.05')
573
+ await mgr.charge('result', 1)
574
+ assert mgr.calculate_max_event_charge_count_within_limit('result') == 4
575
+ assert mgr._platform_other_usd == 0
576
+
577
+
578
+ @pytest.mark.asyncio
579
+ async def test_platform_other_spend_never_shrinks() -> None:
580
+ api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('1'), usage=Decimal('0.30'))
581
+ mgr = _manager(api, max_usd='1')
582
+ await mgr.charge('result', 1)
583
+ seen = mgr._platform_other_usd
584
+ assert seen == Decimal('0.30')
585
+ api.usage = Decimal('0.10') # a stale or lower report
586
+ await mgr.charge('result', 1)
587
+ assert mgr._platform_other_usd == seen