scrapely-python 0.1.17__py3-none-any.whl → 0.1.19__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
scrapely/__init__.py CHANGED
@@ -132,7 +132,7 @@ from .errors import (
132
132
  raise_for_status,
133
133
  )
134
134
 
135
- __version__ = '0.1.17'
135
+ __version__ = '0.1.19'
136
136
  __all__ = [
137
137
  # Core
138
138
  'Actor',
scrapely/actor.py CHANGED
@@ -28,6 +28,7 @@ from .charging import (
28
28
  ChargingManager,
29
29
  is_default_dataset_item_event,
30
30
  push_data_and_charge,
31
+ push_data_with_charges,
31
32
  set_active_charging_manager,
32
33
  )
33
34
  from .configuration import Configuration
@@ -781,6 +782,7 @@ class Actor(metaclass=_ActorMeta):
781
782
  charged_event_name: Optional[str] = None,
782
783
  *,
783
784
  event_name: Optional[str] = None,
785
+ charges: Optional[Dict[str, int]] = None,
784
786
  ) -> ChargeResult:
785
787
  """Push data to the default dataset, charging pay-per-event items.
786
788
 
@@ -796,6 +798,10 @@ class Actor(metaclass=_ActorMeta):
796
798
  (Apify Python SDK name; also accepted positionally).
797
799
  event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
798
800
  kept so existing callers keep working.
801
+ charges: Several events per row, charged atomically: ``{'result': 1, 'page': 3}``
802
+ charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
803
+ only when all of its events were charged; rows the spending limit cannot
804
+ pay for in full are not stored. Cannot be combined with ``charged_event_name``.
799
805
 
800
806
  Returns:
801
807
  ChargeResult describing how many items were charged and pushed.
@@ -818,6 +824,16 @@ class Actor(metaclass=_ActorMeta):
818
824
 
819
825
  items = data if isinstance(data, list) else [data]
820
826
  charging_manager = cls.get_charging_manager()
827
+ if charges is not None:
828
+ if event_name is not None:
829
+ raise ValueError('Pass either charged_event_name or charges, not both')
830
+ return await push_data_with_charges(
831
+ charging_manager,
832
+ items,
833
+ charges,
834
+ True, # Actor.push_data always targets the default dataset
835
+ cls._default_dataset.push_data,
836
+ )
821
837
  return await push_data_and_charge(
822
838
  charging_manager,
823
839
  items,
scrapely/charging.py CHANGED
@@ -240,6 +240,12 @@ class ChargingManager:
240
240
  self._charged_events: Dict[str, int] = {}
241
241
  self._max_total_charge_usd: Optional[Decimal] = None
242
242
  self._total_charged_usd: Decimal = Decimal('0.0')
243
+ # Run spend the platform counts against maxTotalChargeUsd that this SDK
244
+ # does not charge itself: compute/proxy/browser usage and server-billed
245
+ # events. Learnt from the charge endpoint's remainingBudgetNd (only ever
246
+ # grows), so the local budget gates on the run's true remaining budget.
247
+ self._platform_other_usd: Decimal = Decimal('0')
248
+ self._last_remaining_budget_nd: Optional[int] = None
243
249
  self._charge_lock = RLock()
244
250
  self._flush_lock = asyncio.Lock()
245
251
  # Serializes budget calculation/reservation with PPE-aware dataset pushes.
@@ -288,6 +294,27 @@ class ChargingManager:
288
294
  self._refused_default_dataset_items,
289
295
  )
290
296
 
297
+ def _remaining_budget_usd(self) -> Decimal:
298
+ """maxTotalChargeUsd minus this SDK's charges and the platform's other run spend."""
299
+ assert self._max_total_charge_usd is not None
300
+ return self._max_total_charge_usd - self._total_charged_usd - self._platform_other_usd
301
+
302
+ def _sync_platform_budget(self) -> None:
303
+ """Fold the last charge response's remainingBudgetNd into the local budget.
304
+
305
+ Called after the local reservation matches what the platform accepted.
306
+ The platform's remaining budget = max - (our charges + usage + server
307
+ events), so ``other`` = max - remaining - our charges. Monotonic: usage
308
+ never shrinks, and a stale answer must not hand budget back.
309
+ """
310
+ nd = self._last_remaining_budget_nd
311
+ self._last_remaining_budget_nd = None
312
+ if nd is None or self._max_total_charge_usd is None or nd < 0:
313
+ return
314
+ other = self._max_total_charge_usd - Decimal(nd) / Decimal(1_000_000_000) - self._total_charged_usd
315
+ if other > self._platform_other_usd:
316
+ self._platform_other_usd = other
317
+
291
318
  def is_charge_limit_reached(self) -> bool:
292
319
  """Whether this pay-per-event run has used up its spending limit.
293
320
 
@@ -307,7 +334,7 @@ class ChargingManager:
307
334
  prices = [price for price in self._per_event_prices.values() if price > 0]
308
335
  if not prices:
309
336
  return False
310
- return self._max_total_charge_usd - self._total_charged_usd < min(prices)
337
+ return self._remaining_budget_usd() < min(prices)
311
338
 
312
339
  def _mark_charge_limit_hit(self) -> None:
313
340
  if self._pricing_info is not None:
@@ -795,6 +822,7 @@ class ChargingManager:
795
822
  idempotency_key=idempotency_key,
796
823
  deadline=deadline,
797
824
  )
825
+ self._last_remaining_budget_nd = _read_remaining_budget_nd(body)
798
826
  return _read_accepted_count(body, count)
799
827
  except Exception as e:
800
828
  last_error = e
@@ -957,7 +985,7 @@ class ChargingManager:
957
985
  # legacy one-event overdraw while the budget is not yet exceeded.
958
986
  chargeable_count = count
959
987
  if self._max_total_charge_usd is not None and affordability_price > 0:
960
- remaining = self._max_total_charge_usd - self._total_charged_usd
988
+ remaining = self._remaining_budget_usd()
961
989
  max_count_by_limit = max(0, int(remaining / affordability_price))
962
990
  if count > max_count_by_limit:
963
991
  chargeable_count = (
@@ -1045,6 +1073,15 @@ class ChargingManager:
1045
1073
  charged_count=chargeable_count,
1046
1074
  event_charge_limit_reached=self._is_event_limit_reached(event_name),
1047
1075
  )
1076
+ self._sync_platform_budget()
1077
+ if self._max_total_charge_usd is not None and self._platform_other_usd > 0:
1078
+ # Usage may have used up what is left: report it on this result.
1079
+ result = ChargeResult(
1080
+ chargeable_within_limit=self._compute_chargeable(),
1081
+ charged_count=result.charged_count,
1082
+ event_charge_limit_reached=result.event_charge_limit_reached
1083
+ or self._is_event_limit_reached(event_name),
1084
+ )
1048
1085
  else:
1049
1086
  raise ChargeUncertainError(
1050
1087
  f'Charge for event {event_name!r} (count={chargeable_count}) '
@@ -1103,7 +1140,7 @@ class ChargingManager:
1103
1140
  if event_price is None or event_price <= 0:
1104
1141
  return None
1105
1142
 
1106
- remaining = self._max_total_charge_usd - self._total_charged_usd
1143
+ remaining = self._remaining_budget_usd()
1107
1144
  return max(0, int(remaining / event_price))
1108
1145
 
1109
1146
  def calculate_total_charged_amount(self) -> Decimal:
@@ -1165,7 +1202,7 @@ class ChargingManager:
1165
1202
  if self._max_total_charge_usd is None:
1166
1203
  return items_count
1167
1204
 
1168
- remaining = self._max_total_charge_usd - self._total_charged_usd
1205
+ remaining = self._remaining_budget_usd()
1169
1206
  max_items = int(remaining / combined_price)
1170
1207
 
1171
1208
  return min(items_count, max(0, max_items))
@@ -1226,7 +1263,7 @@ class ChargingManager:
1226
1263
  if self._max_total_charge_usd is None:
1227
1264
  return False
1228
1265
 
1229
- remaining = self._max_total_charge_usd - self._total_charged_usd
1266
+ remaining = self._remaining_budget_usd()
1230
1267
  return remaining < event_price
1231
1268
 
1232
1269
  def _compute_chargeable(self) -> Dict[str, Optional[int]]:
@@ -1264,6 +1301,17 @@ def _is_charge_limit_rejection(err: ChargeRejectedError) -> bool:
1264
1301
  )
1265
1302
 
1266
1303
 
1304
+ def _read_remaining_budget_nd(body: Any) -> Optional[int]:
1305
+ """remainingBudgetNd from a charge response (platforms that send it), else None."""
1306
+ if not isinstance(body, dict):
1307
+ return None
1308
+ data = body.get('data') if isinstance(body.get('data'), dict) else body
1309
+ raw = data.get('remainingBudgetNd') if isinstance(data, dict) else None
1310
+ if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw >= 0:
1311
+ return int(raw)
1312
+ return None
1313
+
1314
+
1267
1315
  def _read_accepted_count(body: Any, fallback: int) -> int:
1268
1316
  if not isinstance(body, dict):
1269
1317
  return fallback
@@ -1585,6 +1633,166 @@ async def push_data_and_charge(
1585
1633
  )
1586
1634
 
1587
1635
 
1636
+ def _validate_row_charges(charges: Any) -> Dict[str, int]:
1637
+ if not isinstance(charges, dict) or not charges:
1638
+ raise ValueError('charges must be a non-empty dict of {event_name: count_per_row}')
1639
+ out: Dict[str, int] = {}
1640
+ for name, count in charges.items():
1641
+ if not isinstance(name, str) or not name:
1642
+ raise ValueError('charges keys must be event names')
1643
+ if is_default_dataset_item_event(name):
1644
+ raise ValueError(f"Cannot charge for synthetic event '{name}' manually")
1645
+ if not isinstance(count, int) or isinstance(count, bool) or count < 1:
1646
+ raise ValueError(f'charges[{name!r}] must be a positive integer (events per row)')
1647
+ out[name] = count
1648
+ return out
1649
+
1650
+
1651
+ async def push_data_with_charges(
1652
+ charging_manager: ChargingManager,
1653
+ items: Union[Dict[str, Any], List[Dict[str, Any]]],
1654
+ charges: Dict[str, int],
1655
+ is_default_dataset: bool,
1656
+ push_fn: Any,
1657
+ ) -> ChargeResult:
1658
+ """Push rows that each cost several events, atomically per row.
1659
+
1660
+ ``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
1661
+ A row is stored only when every one of its events was charged: the full set
1662
+ is charged first, rows the budget cannot fully pay for are not stored, and
1663
+ any operation that came back short is voided and re-charged for the rows
1664
+ that are kept, so no row is ever stored partly paid and no event is billed
1665
+ for a row that was not stored. Unpriced events are ignored. On the default
1666
+ dataset the default-dataset-item event (when priced) is added per row.
1667
+ """
1668
+ charges = _validate_row_charges(charges)
1669
+ rows_in = items if isinstance(items, list) else [items]
1670
+ if not rows_in:
1671
+ return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
1672
+
1673
+ async with charging_manager._operation_lock:
1674
+ if charging_manager._closed:
1675
+ raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
1676
+ voids_confirmed = await charging_manager._retry_pending_voids()
1677
+
1678
+ if charging_manager._pricing_info is None:
1679
+ await _gated_push(push_fn, rows_in)
1680
+ return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
1681
+
1682
+ per_row: Dict[str, int] = {
1683
+ name: n for name, n in charges.items()
1684
+ if (charging_manager._calculate_event_price(name) or Decimal('0')) > 0
1685
+ }
1686
+ if is_default_dataset:
1687
+ default_event = charging_manager._config.get('default_dataset_item_event_name') or \
1688
+ resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
1689
+ if default_event and default_event not in per_row and \
1690
+ (charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
1691
+ per_row[default_event] = 1
1692
+ if not per_row:
1693
+ # Every listed event is free: a plain (unpaid) push.
1694
+ return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
1695
+ if not voids_confirmed and any(not is_default_dataset_item_event(n) for n in per_row):
1696
+ raise RuntimeError('An earlier charge for a failed dataset push could not be voided yet; retry the push')
1697
+
1698
+ # Rows the budget can pay for in full (no one-event overdraw: a row is all or nothing).
1699
+ row_cost = sum(
1700
+ (charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
1701
+ )
1702
+ rows = len(rows_in)
1703
+ if charging_manager._max_total_charge_usd is not None and row_cost > 0:
1704
+ remaining = charging_manager._remaining_budget_usd()
1705
+ rows = min(rows, max(0, int(remaining / row_cost)))
1706
+
1707
+ ops: List[tuple] = [] # (event, charged, op_id, uncertain)
1708
+
1709
+ async def undo(entries: List[tuple]) -> None:
1710
+ for name, charged, op_id, uncertain in entries:
1711
+ if charged <= 0:
1712
+ continue
1713
+ if op_id is None:
1714
+ charging_manager._unwind_reservation(name, charged)
1715
+ elif uncertain:
1716
+ await charging_manager._cancel_uncertain_charge(name, op_id, charged)
1717
+ else:
1718
+ await charging_manager._void_charge(name, op_id, charged)
1719
+
1720
+ for _attempt in range(4):
1721
+ if rows <= 0:
1722
+ break
1723
+ ops = []
1724
+ short_to: Optional[int] = None
1725
+ for name, n in sorted(per_row.items(), key=lambda kv: (is_default_dataset_item_event(kv[0]), kv[0])):
1726
+ want = n * rows
1727
+ charging_manager._take_last_accepted_operation_id()
1728
+ try:
1729
+ r = await charging_manager._charge_exclusive(name, want)
1730
+ op_id = charging_manager._take_last_accepted_operation_id()
1731
+ ops.append((name, r.charged_count, op_id, False))
1732
+ got = r.charged_count
1733
+ except ChargeUncertainError as unconfirmed:
1734
+ # Not confirmed: the intent stays pending and is re-posted at exit,
1735
+ # so it counts as accepted (items stored, billed once).
1736
+ ops.append((name, unconfirmed.count, unconfirmed.operation_id, True))
1737
+ got = unconfirmed.count
1738
+ if got < want:
1739
+ short_to = got // n
1740
+ break
1741
+ if short_to is None:
1742
+ break
1743
+ # Some event came back short: void this attempt in full and retry with
1744
+ # the rows that event can pay for (a void restores the budget).
1745
+ await undo(ops)
1746
+ ops = []
1747
+ rows = short_to # got < n * rows, so short_to < rows
1748
+
1749
+ if rows <= 0 or not ops:
1750
+ await undo(ops)
1751
+ charging_manager._mark_charge_limit_hit()
1752
+ if is_default_dataset:
1753
+ charging_manager._note_refused_default_dataset_items(len(rows_in))
1754
+ return ChargeResult(
1755
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1756
+ charged_count=0,
1757
+ event_charge_limit_reached=True,
1758
+ )
1759
+
1760
+ kept = rows_in[:rows]
1761
+ if rows < len(rows_in):
1762
+ charging_manager._mark_charge_limit_hit()
1763
+ if is_default_dataset:
1764
+ charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
1765
+ try:
1766
+ await _gated_push(push_fn, kept)
1767
+ except Exception as push_err:
1768
+ logger.error('Dataset push failed after charges accepted (rows=%s): %s; voiding the charges', rows, push_err)
1769
+ if _push_certainly_failed(push_err):
1770
+ await undo(ops)
1771
+ raise
1772
+ return ChargeResult(
1773
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1774
+ charged_count=rows,
1775
+ event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
1776
+ )
1777
+
1778
+
1779
+ async def _unpaid_push_locked(
1780
+ charging_manager: ChargingManager, rows: List[Dict[str, Any]], is_default_dataset: bool, push_fn: Any
1781
+ ) -> ChargeResult:
1782
+ """A push whose rows carry no priced event (caller holds the operation lock)."""
1783
+ past_limit = charging_manager.is_charge_limit_reached()
1784
+ allowed = charging_manager._take_unpaid_rows(len(rows)) if is_default_dataset else len(rows)
1785
+ if allowed < len(rows):
1786
+ charging_manager._note_refused_default_dataset_items(len(rows) - allowed)
1787
+ if allowed > 0:
1788
+ await _gated_push(push_fn, rows[:allowed])
1789
+ return ChargeResult(
1790
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1791
+ charged_count=0,
1792
+ event_charge_limit_reached=past_limit,
1793
+ )
1794
+
1795
+
1588
1796
  def _push_is_paid_per_item(
1589
1797
  charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
1590
1798
  ) -> bool:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-python
3
- Version: 0.1.17
3
+ Version: 0.1.19
4
4
  Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
5
5
  Author-email: Scrapely <support@scrape.ly>
6
6
  License-Expression: MIT AND Apache-2.0
@@ -1,9 +1,9 @@
1
- scrapely/__init__.py,sha256=TqM8pFfiE1_j1M69_2BOtRVAy1jMK6HzvKNpmYtNnuo,5746
1
+ scrapely/__init__.py,sha256=7AUWixuhrohpSFpDkEHo809IBKI-2u-m4ZWPMorYKSk,5746
2
2
  scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
3
- scrapely/actor.py,sha256=1MXGyHql5sRfh15HMRY8vAp_TNr_yhavDALlHBcjLO0,80810
3
+ scrapely/actor.py,sha256=Lsb4_14RCGQy-EWACnqijo7Hz5xvPKSDZcu4B8_ZrHc,81685
4
4
  scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
5
5
  scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
6
- scrapely/charging.py,sha256=ImnjF4RfzLbpRp3JZIl2RM-TfHp22p0Xzt0NSVs2M1c,71477
6
+ scrapely/charging.py,sha256=TxcALWN4hhwJ0i2raYmtHGA1P79lQ1bmhBMtrvNACIw,81862
7
7
  scrapely/charging_client.py,sha256=-4q-Ww7gu9EZXpM9QIn1Vd4vwAXmgL_WEKqaQIx3ViA,3961
8
8
  scrapely/configuration.py,sha256=Of-lKm7hoPGUQO8d08kV7fAy2co9QTfb8Of_8ds5PSA,17028
9
9
  scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
@@ -39,10 +39,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
39
39
  scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
40
40
  scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
41
41
  scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
42
- scrapely_python-0.1.17.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
43
- scrapely_python-0.1.17.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
44
- scrapely_python-0.1.17.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
45
- scrapely_python-0.1.17.dist-info/METADATA,sha256=CUcmUdVmdGoDotCHQvZUa2eYIYKugeJZvsbA9tWrVRg,3487
46
- scrapely_python-0.1.17.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
47
- scrapely_python-0.1.17.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
48
- scrapely_python-0.1.17.dist-info/RECORD,,
42
+ scrapely_python-0.1.19.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
43
+ scrapely_python-0.1.19.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
44
+ scrapely_python-0.1.19.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
45
+ scrapely_python-0.1.19.dist-info/METADATA,sha256=PzFQ4BxF0FgSjgOvSQYimfc5v6Pq2notGIWYHCBa-iQ,3487
46
+ scrapely_python-0.1.19.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
47
+ scrapely_python-0.1.19.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
48
+ scrapely_python-0.1.19.dist-info/RECORD,,