scrapely-python 0.1.16__tar.gz → 0.1.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. {scrapely_python-0.1.16/src/scrapely_python.egg-info → scrapely_python-0.1.18}/PKG-INFO +1 -1
  2. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/pyproject.toml +1 -1
  3. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/__init__.py +1 -1
  4. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/actor.py +16 -0
  5. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/charging.py +221 -11
  6. {scrapely_python-0.1.16 → scrapely_python-0.1.18/src/scrapely_python.egg-info}/PKG-INFO +1 -1
  7. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_charge_limit_guard.py +220 -0
  8. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/LICENSE +0 -0
  9. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/LICENSE-APACHE +0 -0
  10. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/NOTICE +0 -0
  11. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/README.md +0 -0
  12. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/setup.cfg +0 -0
  13. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/_browser_locations.py +0 -0
  14. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/browser_configuration.py +0 -0
  15. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/cdp_retry.py +0 -0
  16. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/charging_client.py +0 -0
  17. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/configuration.py +0 -0
  18. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/crypto/__init__.py +0 -0
  19. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/crypto/_secrets.py +0 -0
  20. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/errors.py +0 -0
  21. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/events/__init__.py +0 -0
  22. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/events/_types.py +0 -0
  23. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/events/event_manager.py +0 -0
  24. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/log.py +0 -0
  25. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/patch_browser_pool.py +0 -0
  26. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/pricing_types.py +0 -0
  27. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/proxy_configuration.py +0 -0
  28. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/py.typed +0 -0
  29. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/request_loaders/__init__.py +0 -0
  30. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/request_loaders/_scrapely_request_list.py +0 -0
  31. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/solver.py +0 -0
  32. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/__init__.py +0 -0
  33. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/_file_system/__init__.py +0 -0
  34. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/_file_system/_key_value_store_client.py +0 -0
  35. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/_file_system/_storage_client.py +0 -0
  36. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/_metadata.py +0 -0
  37. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/api_client.py +0 -0
  38. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/client.py +0 -0
  39. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/patch_storage.py +0 -0
  40. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_dataset_client.py +0 -0
  41. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_kvs_client.py +0 -0
  42. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_rq_client.py +0 -0
  43. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_storage_client.py +0 -0
  44. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/smart_scrapely_storage_client.py +0 -0
  45. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/types.py +0 -0
  46. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/utils.py +0 -0
  47. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage_clients/__init__.py +0 -0
  48. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storages/__init__.py +0 -0
  49. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/types.py +0 -0
  50. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/webhook.py +0 -0
  51. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/SOURCES.txt +0 -0
  52. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/dependency_links.txt +0 -0
  53. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/requires.txt +0 -0
  54. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/top_level.txt +0 -0
  55. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_api_client_retries.py +0 -0
  56. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_apify_parity.py +0 -0
  57. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_browser_configuration.py +0 -0
  58. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_cdp_retry.py +0 -0
  59. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_charge_retries.py +0 -0
  60. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_charging_parity.py +0 -0
  61. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_charging_pending.py +0 -0
  62. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_input_proxy_aliases.py +0 -0
  63. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_parity.py +0 -0
  64. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_patch_browser_pool.py +0 -0
  65. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_proxy_configuration.py +0 -0
  66. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_push_idempotency.py +0 -0
  67. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_run_control.py +0 -0
  68. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_secrets.py +0 -0
  69. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_solver.py +0 -0
  70. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_storage.py +0 -0
  71. {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_terminal_status_message.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-python
3
- Version: 0.1.16
3
+ Version: 0.1.18
4
4
  Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
5
5
  Author-email: Scrapely <support@scrape.ly>
6
6
  License-Expression: MIT AND Apache-2.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "scrapely-python"
7
- version = "0.1.16"
7
+ version = "0.1.18"
8
8
  description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
9
9
  readme = "README.md"
10
10
  license = "MIT AND Apache-2.0"
@@ -132,7 +132,7 @@ from .errors import (
132
132
  raise_for_status,
133
133
  )
134
134
 
135
- __version__ = '0.1.16'
135
+ __version__ = '0.1.18'
136
136
  __all__ = [
137
137
  # Core
138
138
  'Actor',
@@ -28,6 +28,7 @@ from .charging import (
28
28
  ChargingManager,
29
29
  is_default_dataset_item_event,
30
30
  push_data_and_charge,
31
+ push_data_with_charges,
31
32
  set_active_charging_manager,
32
33
  )
33
34
  from .configuration import Configuration
@@ -781,6 +782,7 @@ class Actor(metaclass=_ActorMeta):
781
782
  charged_event_name: Optional[str] = None,
782
783
  *,
783
784
  event_name: Optional[str] = None,
785
+ charges: Optional[Dict[str, int]] = None,
784
786
  ) -> ChargeResult:
785
787
  """Push data to the default dataset, charging pay-per-event items.
786
788
 
@@ -796,6 +798,10 @@ class Actor(metaclass=_ActorMeta):
796
798
  (Apify Python SDK name; also accepted positionally).
797
799
  event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
798
800
  kept so existing callers keep working.
801
+ charges: Several events per row, charged atomically: ``{'result': 1, 'page': 3}``
802
+ charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
803
+ only when all of its events were charged; rows the spending limit cannot
804
+ pay for in full are not stored. Cannot be combined with ``charged_event_name``.
799
805
 
800
806
  Returns:
801
807
  ChargeResult describing how many items were charged and pushed.
@@ -818,6 +824,16 @@ class Actor(metaclass=_ActorMeta):
818
824
 
819
825
  items = data if isinstance(data, list) else [data]
820
826
  charging_manager = cls.get_charging_manager()
827
+ if charges is not None:
828
+ if event_name is not None:
829
+ raise ValueError('Pass either charged_event_name or charges, not both')
830
+ return await push_data_with_charges(
831
+ charging_manager,
832
+ items,
833
+ charges,
834
+ True, # Actor.push_data always targets the default dataset
835
+ cls._default_dataset.push_data,
836
+ )
821
837
  return await push_data_and_charge(
822
838
  charging_manager,
823
839
  items,
@@ -270,6 +270,13 @@ class ChargingManager:
270
270
  self._charge_limit_hit = False
271
271
  # Default-dataset rows refused after the limit (one warning, then a count).
272
272
  self._refused_default_dataset_items = 0
273
+ # Prepaid-row balance: priced events charged through charge() minus rows
274
+ # pushed to the default dataset without a per-row charge. An Actor that
275
+ # charges first and then pushes plainly (charge('product'); push_data(row))
276
+ # has a positive balance after the charge, so the row it already paid for
277
+ # is still saved when that charge used up the spending limit. Push-then-
278
+ # charge Actors never go positive, so they get nothing extra for free.
279
+ self._prepaid_row_balance = 0
273
280
 
274
281
  def close(self) -> None:
275
282
  """Mark the manager closed; subsequent charge/push operations are rejected."""
@@ -323,8 +330,40 @@ class ChargingManager:
323
330
  )
324
331
  self._refused_default_dataset_items += count
325
332
 
333
+ def _note_prepaid_charge(self, event_name: str, charged_count: int) -> None:
334
+ if (
335
+ charged_count > 0
336
+ and not is_default_dataset_item_event(event_name)
337
+ and self._per_event_prices.get(event_name, 0) > 0
338
+ ):
339
+ self._prepaid_row_balance += charged_count
340
+
341
+ def _take_unpaid_rows(self, item_count: int) -> int:
342
+ """Account for ``item_count`` rows pushed without a per-row charge.
343
+
344
+ Returns how many of them may be stored: all of them while the limit is
345
+ not reached, afterwards only the rows already paid for by earlier
346
+ charges (the positive prepaid balance).
347
+ """
348
+ if item_count <= 0:
349
+ return 0
350
+ if not self.is_charge_limit_reached():
351
+ self._prepaid_row_balance -= item_count
352
+ return item_count
353
+ allowed = max(0, min(item_count, self._prepaid_row_balance))
354
+ self._prepaid_row_balance -= allowed
355
+ return allowed
356
+
326
357
  def _refuse_default_dataset_push(self, item_count: int) -> bool:
327
- if item_count <= 0 or not self.is_charge_limit_reached():
358
+ if item_count <= 0 or self._pricing_info is None:
359
+ return False
360
+ if not self.is_charge_limit_reached():
361
+ self._take_unpaid_rows(item_count)
362
+ return False
363
+ # All-or-nothing here (storage clients cannot store a slice): store the
364
+ # batch only when earlier charges paid for every row of it.
365
+ if self._prepaid_row_balance >= item_count:
366
+ self._prepaid_row_balance -= item_count
328
367
  return False
329
368
  self._note_refused_default_dataset_items(item_count)
330
369
  return True
@@ -851,7 +890,9 @@ class ChargingManager:
851
890
  async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
852
891
  """Atomically reserve and submit one logical charge operation."""
853
892
  async with self._operation_lock:
854
- return await self._charge_exclusive(event_name, count)
893
+ result = await self._charge_exclusive(event_name, count)
894
+ self._note_prepaid_charge(event_name, result.charged_count)
895
+ return result
855
896
 
856
897
  async def _charge_exclusive(self, event_name: str, count: int = 1) -> ChargeResult:
857
898
  if self._closed:
@@ -1327,19 +1368,27 @@ async def push_data_and_charge(
1327
1368
  # default-dataset push that does not pay per row (no event, or only
1328
1369
  # free events) stores nothing. Paid pushes go on below, where the
1329
1370
  # budget cap stores only the rows that are actually charged.
1371
+ # Rows the Actor already paid for with an earlier charge() (charge first,
1372
+ # then a plain push) are still stored: see ChargingManager._take_unpaid_rows.
1373
+ # The result still reports the limit, so the Actor stops after them.
1374
+ unpaid_past_limit = False
1330
1375
  if (
1331
1376
  is_ppe
1332
1377
  and is_default_dataset
1333
1378
  and items
1334
- and charging_manager.is_charge_limit_reached()
1335
1379
  and not _push_is_paid_per_item(charging_manager, event_name, resolved_event)
1336
1380
  ):
1337
- charging_manager._note_refused_default_dataset_items(len(items))
1338
- return ChargeResult(
1339
- chargeable_within_limit=charging_manager.compute_chargeable(),
1340
- charged_count=0,
1341
- event_charge_limit_reached=True,
1342
- )
1381
+ unpaid_past_limit = charging_manager.is_charge_limit_reached()
1382
+ allowed = charging_manager._take_unpaid_rows(len(items))
1383
+ if allowed < len(items):
1384
+ charging_manager._note_refused_default_dataset_items(len(items) - allowed)
1385
+ if allowed == 0:
1386
+ return ChargeResult(
1387
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1388
+ charged_count=0,
1389
+ event_charge_limit_reached=True,
1390
+ )
1391
+ items = items[:allowed]
1343
1392
 
1344
1393
  limited_count = len(items)
1345
1394
  if is_ppe and resolved_event:
@@ -1522,7 +1571,8 @@ async def push_data_and_charge(
1522
1571
  event_charge_limit_reached=any(
1523
1572
  result.event_charge_limit_reached for result in results
1524
1573
  )
1525
- or accepted_for_items < len(limited_items),
1574
+ or accepted_for_items < len(limited_items)
1575
+ or unpaid_past_limit,
1526
1576
  )
1527
1577
 
1528
1578
 
@@ -1531,10 +1581,170 @@ async def push_data_and_charge(
1531
1581
  return ChargeResult(
1532
1582
  chargeable_within_limit=charging_manager.compute_chargeable(),
1533
1583
  charged_count=0,
1534
- event_charge_limit_reached=False,
1584
+ event_charge_limit_reached=unpaid_past_limit,
1535
1585
  )
1536
1586
 
1537
1587
 
1588
+ def _validate_row_charges(charges: Any) -> Dict[str, int]:
1589
+ if not isinstance(charges, dict) or not charges:
1590
+ raise ValueError('charges must be a non-empty dict of {event_name: count_per_row}')
1591
+ out: Dict[str, int] = {}
1592
+ for name, count in charges.items():
1593
+ if not isinstance(name, str) or not name:
1594
+ raise ValueError('charges keys must be event names')
1595
+ if is_default_dataset_item_event(name):
1596
+ raise ValueError(f"Cannot charge for synthetic event '{name}' manually")
1597
+ if not isinstance(count, int) or isinstance(count, bool) or count < 1:
1598
+ raise ValueError(f'charges[{name!r}] must be a positive integer (events per row)')
1599
+ out[name] = count
1600
+ return out
1601
+
1602
+
1603
+ async def push_data_with_charges(
1604
+ charging_manager: ChargingManager,
1605
+ items: Union[Dict[str, Any], List[Dict[str, Any]]],
1606
+ charges: Dict[str, int],
1607
+ is_default_dataset: bool,
1608
+ push_fn: Any,
1609
+ ) -> ChargeResult:
1610
+ """Push rows that each cost several events, atomically per row.
1611
+
1612
+ ``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
1613
+ A row is stored only when every one of its events was charged: the full set
1614
+ is charged first, rows the budget cannot fully pay for are not stored, and
1615
+ any operation that came back short is voided and re-charged for the rows
1616
+ that are kept, so no row is ever stored partly paid and no event is billed
1617
+ for a row that was not stored. Unpriced events are ignored. On the default
1618
+ dataset the default-dataset-item event (when priced) is added per row.
1619
+ """
1620
+ charges = _validate_row_charges(charges)
1621
+ rows_in = items if isinstance(items, list) else [items]
1622
+ if not rows_in:
1623
+ return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
1624
+
1625
+ async with charging_manager._operation_lock:
1626
+ if charging_manager._closed:
1627
+ raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
1628
+ voids_confirmed = await charging_manager._retry_pending_voids()
1629
+
1630
+ if charging_manager._pricing_info is None:
1631
+ await _gated_push(push_fn, rows_in)
1632
+ return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
1633
+
1634
+ per_row: Dict[str, int] = {
1635
+ name: n for name, n in charges.items()
1636
+ if (charging_manager._calculate_event_price(name) or Decimal('0')) > 0
1637
+ }
1638
+ if is_default_dataset:
1639
+ default_event = charging_manager._config.get('default_dataset_item_event_name') or \
1640
+ resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
1641
+ if default_event and default_event not in per_row and \
1642
+ (charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
1643
+ per_row[default_event] = 1
1644
+ if not per_row:
1645
+ # Every listed event is free: a plain (unpaid) push.
1646
+ return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
1647
+ if not voids_confirmed and any(not is_default_dataset_item_event(n) for n in per_row):
1648
+ raise RuntimeError('An earlier charge for a failed dataset push could not be voided yet; retry the push')
1649
+
1650
+ # Rows the budget can pay for in full (no one-event overdraw: a row is all or nothing).
1651
+ row_cost = sum(
1652
+ (charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
1653
+ )
1654
+ rows = len(rows_in)
1655
+ if charging_manager._max_total_charge_usd is not None and row_cost > 0:
1656
+ remaining = charging_manager._max_total_charge_usd - charging_manager._total_charged_usd
1657
+ rows = min(rows, max(0, int(remaining / row_cost)))
1658
+
1659
+ ops: List[tuple] = [] # (event, charged, op_id, uncertain)
1660
+
1661
+ async def undo(entries: List[tuple]) -> None:
1662
+ for name, charged, op_id, uncertain in entries:
1663
+ if charged <= 0:
1664
+ continue
1665
+ if op_id is None:
1666
+ charging_manager._unwind_reservation(name, charged)
1667
+ elif uncertain:
1668
+ await charging_manager._cancel_uncertain_charge(name, op_id, charged)
1669
+ else:
1670
+ await charging_manager._void_charge(name, op_id, charged)
1671
+
1672
+ for _attempt in range(4):
1673
+ if rows <= 0:
1674
+ break
1675
+ ops = []
1676
+ short_to: Optional[int] = None
1677
+ for name, n in sorted(per_row.items(), key=lambda kv: (is_default_dataset_item_event(kv[0]), kv[0])):
1678
+ want = n * rows
1679
+ charging_manager._take_last_accepted_operation_id()
1680
+ try:
1681
+ r = await charging_manager._charge_exclusive(name, want)
1682
+ op_id = charging_manager._take_last_accepted_operation_id()
1683
+ ops.append((name, r.charged_count, op_id, False))
1684
+ got = r.charged_count
1685
+ except ChargeUncertainError as unconfirmed:
1686
+ # Not confirmed: the intent stays pending and is re-posted at exit,
1687
+ # so it counts as accepted (items stored, billed once).
1688
+ ops.append((name, unconfirmed.count, unconfirmed.operation_id, True))
1689
+ got = unconfirmed.count
1690
+ if got < want:
1691
+ short_to = got // n
1692
+ break
1693
+ if short_to is None:
1694
+ break
1695
+ # Some event came back short: void this attempt in full and retry with
1696
+ # the rows that event can pay for (a void restores the budget).
1697
+ await undo(ops)
1698
+ ops = []
1699
+ rows = short_to # got < n * rows, so short_to < rows
1700
+
1701
+ if rows <= 0 or not ops:
1702
+ await undo(ops)
1703
+ charging_manager._mark_charge_limit_hit()
1704
+ if is_default_dataset:
1705
+ charging_manager._note_refused_default_dataset_items(len(rows_in))
1706
+ return ChargeResult(
1707
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1708
+ charged_count=0,
1709
+ event_charge_limit_reached=True,
1710
+ )
1711
+
1712
+ kept = rows_in[:rows]
1713
+ if rows < len(rows_in):
1714
+ charging_manager._mark_charge_limit_hit()
1715
+ if is_default_dataset:
1716
+ charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
1717
+ try:
1718
+ await _gated_push(push_fn, kept)
1719
+ except Exception as push_err:
1720
+ logger.error('Dataset push failed after charges accepted (rows=%s): %s; voiding the charges', rows, push_err)
1721
+ if _push_certainly_failed(push_err):
1722
+ await undo(ops)
1723
+ raise
1724
+ return ChargeResult(
1725
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1726
+ charged_count=rows,
1727
+ event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
1728
+ )
1729
+
1730
+
1731
+ async def _unpaid_push_locked(
1732
+ charging_manager: ChargingManager, rows: List[Dict[str, Any]], is_default_dataset: bool, push_fn: Any
1733
+ ) -> ChargeResult:
1734
+ """A push whose rows carry no priced event (caller holds the operation lock)."""
1735
+ past_limit = charging_manager.is_charge_limit_reached()
1736
+ allowed = charging_manager._take_unpaid_rows(len(rows)) if is_default_dataset else len(rows)
1737
+ if allowed < len(rows):
1738
+ charging_manager._note_refused_default_dataset_items(len(rows) - allowed)
1739
+ if allowed > 0:
1740
+ await _gated_push(push_fn, rows[:allowed])
1741
+ return ChargeResult(
1742
+ chargeable_within_limit=charging_manager.compute_chargeable(),
1743
+ charged_count=0,
1744
+ event_charge_limit_reached=past_limit,
1745
+ )
1746
+
1747
+
1538
1748
  def _push_is_paid_per_item(
1539
1749
  charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
1540
1750
  ) -> bool:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-python
3
- Version: 0.1.16
3
+ Version: 0.1.18
4
4
  Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
5
5
  Author-email: Scrapely <support@scrape.ly>
6
6
  License-Expression: MIT AND Apache-2.0
@@ -289,3 +289,223 @@ async def test_apify_client_hooks_intercept_default_dataset_past_limit() -> None
289
289
  assert hooks.should_charge_dataset_pushes(DEFAULT_DATASET_ID) is False
290
290
  await mgr.charge('result', 1)
291
291
  assert hooks.should_charge_dataset_pushes(DEFAULT_DATASET_ID) is True
292
+
293
+
294
+ # ---- charge first, then a plain push (the row was paid before it was pushed) ----
295
+
296
+
297
+ @pytest.mark.asyncio
298
+ async def test_charge_then_plain_push_keeps_the_row_whose_charge_hit_the_cap() -> None:
299
+ """charge('result'); push_data(row): the last paid row must be saved, nothing after it."""
300
+ api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
301
+ mgr = _manager(api)
302
+ store = Store()
303
+
304
+ for n in range(10):
305
+ r = await mgr.charge('result', 1)
306
+ if r.charged_count:
307
+ await _plain_push(mgr, store, [{'n': n}])
308
+ else:
309
+ await _plain_push(mgr, store, [{'n': n, 'unpaid': True}])
310
+
311
+ assert [i['n'] for i in store.items] == [0, 1, 2]
312
+ assert api.spent == Decimal('0.03')
313
+ assert mgr.is_charge_limit_reached() is True
314
+
315
+
316
+ @pytest.mark.asyncio
317
+ async def test_prepaid_rows_cap_a_bigger_plain_push() -> None:
318
+ api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.02'))
319
+ mgr = _manager(api, max_usd='0.02')
320
+ store = Store()
321
+
322
+ await mgr.charge('result', 2)
323
+ assert mgr.is_charge_limit_reached() is True
324
+ r = await _plain_push(mgr, store, [{'n': 1}, {'n': 2}, {'n': 3}])
325
+ assert [i['n'] for i in store.items] == [1, 2]
326
+ assert r.event_charge_limit_reached is True
327
+ assert mgr._refused_default_dataset_items == 1
328
+ # Balance used up: the next plain row is refused.
329
+ await _plain_push(mgr, store, [{'n': 4}])
330
+ assert len(store.items) == 2
331
+
332
+
333
+ @pytest.mark.asyncio
334
+ async def test_push_then_charge_actor_gets_no_free_row_after_the_cap() -> None:
335
+ """Balance never goes positive for push-then-charge, so the cap still stops it."""
336
+ api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
337
+ mgr = _manager(api)
338
+ store = Store()
339
+
340
+ for n in range(6):
341
+ await _plain_push(mgr, store, [{'n': n}])
342
+ await mgr.charge('result', 1)
343
+
344
+ assert [i['n'] for i in store.items] == [0, 1, 2]
345
+ assert mgr._prepaid_row_balance <= 0
346
+
347
+
348
+ @pytest.mark.asyncio
349
+ async def test_unpriced_or_free_charges_do_not_prepay_rows() -> None:
350
+ api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
351
+ mgr = _manager(api, prices={'result': Decimal('0.01'), 'free-event': Decimal('0')}, max_usd='0.01')
352
+ store = Store()
353
+
354
+ await mgr.charge('result', 1)
355
+ await _plain_push(mgr, store, [{'n': 1}]) # paid by the charge above
356
+ await mgr.charge('free-event', 3)
357
+ await mgr.charge('not-in-pricing', 3)
358
+ await _plain_push(mgr, store, [{'n': 2}])
359
+ assert [i['n'] for i in store.items] == [1]
360
+
361
+
362
+ @pytest.mark.asyncio
363
+ async def test_storage_client_push_after_prepaying_charge_is_stored() -> None:
364
+ api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
365
+ mgr = _manager(api, max_usd='0.01')
366
+ set_active_charging_manager(mgr)
367
+ client, pushed = _dataset_client(DEFAULT_DATASET_ID)
368
+
369
+ await mgr.charge('result', 1) # uses the whole budget, pays for one row
370
+ await client.push_data([{'n': 1}, {'n': 2}]) # two rows, one paid: whole batch refused
371
+ await client.push_data([{'n': 3}]) # one paid row: stored
372
+ await client.push_data([{'n': 4}]) # nothing left: refused
373
+
374
+ assert pushed == [(DEFAULT_DATASET_ID, [{'n': 3}])]
375
+
376
+
377
+ @pytest.mark.asyncio
378
+ async def test_partial_acceptance_saves_only_the_accepted_rows() -> None:
379
+ api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
380
+ mgr = _manager(api, max_usd='5')
381
+ store = Store()
382
+
383
+ c = await mgr.charge('result', 5)
384
+ assert c.charged_count == 3
385
+ r = await _plain_push(mgr, store, [{'n': n} for n in range(1, 6)])
386
+ assert [i['n'] for i in store.items] == [1, 2, 3]
387
+ assert r.event_charge_limit_reached is True
388
+ await _plain_push(mgr, store, [{'n': 6}])
389
+ assert len(store.items) == 3
390
+
391
+
392
+ @pytest.mark.asyncio
393
+ async def test_refused_charge_402_then_plain_push_is_not_saved() -> None:
394
+ api = FakeChargeApi(price=Decimal('0.01'), max_usd=None, reject_after=2)
395
+ mgr = _manager(api, max_usd='5')
396
+ store = Store()
397
+
398
+ for n in range(4):
399
+ await mgr.charge('result', 1)
400
+ await _plain_push(mgr, store, [{'n': n}])
401
+
402
+ assert [i['n'] for i in store.items] == [0, 1]
403
+
404
+
405
+ # ---- atomic multi-event push: push_data(row, charges={...}) ----
406
+
407
+ from scrapely.charging import push_data_with_charges # noqa: E402
408
+
409
+
410
+ class MultiApi(FakeChargeApi):
411
+ """Charge endpoint priced per event, with void support (by Idempotency-Key)."""
412
+
413
+ def __init__(self, prices: Dict[str, Decimal], max_usd: Optional[Decimal]) -> None:
414
+ super().__init__(price=Decimal('0'), max_usd=max_usd)
415
+ self.prices = prices
416
+ self.ops: Dict[str, tuple] = {}
417
+ self.voids: List[str] = []
418
+
419
+ async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **_: Any) -> Dict[str, Any]:
420
+ key = (headers or {}).get('Idempotency-Key', '')
421
+ if path.endswith('/charge/void'):
422
+ name, accepted = self.ops.pop(key)
423
+ self.spent -= self.prices[name] * accepted
424
+ self.voids.append(name)
425
+ return {}
426
+ name = (json_data or {}).get('eventName')
427
+ count = int((json_data or {}).get('count', 1))
428
+ price = self.prices[name]
429
+ accepted = count
430
+ if self.max_usd is not None and price > 0:
431
+ affordable = int((self.max_usd - self.spent) / price)
432
+ if affordable < 1:
433
+ raise HttpError(402, 'charge-limit-exceeded')
434
+ accepted = min(count, affordable)
435
+ self.spent += price * accepted
436
+ self.ops[key] = (name, accepted)
437
+ return {'acceptedCount': accepted, 'requestedCount': count}
438
+
439
+
440
+ PRICES = {'result': Decimal('0.01'), 'page': Decimal('0.002')} # one row = 1 result + 3 pages = $0.016
441
+
442
+
443
+ def _multi(max_usd: str, server_max: Optional[str] = None):
444
+ api = MultiApi(PRICES, Decimal(server_max or max_usd))
445
+ return api, _manager(api, prices=dict(PRICES), max_usd=max_usd)
446
+
447
+
448
+ @pytest.mark.asyncio
449
+ async def test_multi_event_exact_budget_stores_every_paid_row() -> None:
450
+ api, mgr = _multi('0.048') # exactly 3 rows
451
+ store = Store()
452
+ r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
453
+ assert r.charged_count == 3
454
+ assert [i['n'] for i in store.items] == [0, 1, 2]
455
+ assert api.spent == Decimal('0.048')
456
+ assert mgr.is_charge_limit_reached() is True
457
+ again = await push_data_with_charges(mgr, [{'n': 9}], {'result': 1, 'page': 3}, True, store.push)
458
+ assert again.charged_count == 0 and again.event_charge_limit_reached is True
459
+ assert len(store.items) == 3 and api.spent == Decimal('0.048')
460
+
461
+
462
+ @pytest.mark.asyncio
463
+ async def test_multi_event_partial_budget_stores_only_fully_paid_rows() -> None:
464
+ api, mgr = _multi('0.040') # 2.5 rows: only 2 fully paid
465
+ store = Store()
466
+ r = await push_data_with_charges(mgr, [{'n': n} for n in range(5)], {'result': 1, 'page': 3}, True, store.push)
467
+ assert r.charged_count == 2 and r.event_charge_limit_reached is True
468
+ assert [i['n'] for i in store.items] == [0, 1]
469
+ assert api.spent == Decimal('0.032') # never a part-paid row
470
+
471
+
472
+ @pytest.mark.asyncio
473
+ async def test_multi_event_server_cap_short_voids_and_recharges() -> None:
474
+ """Local budget says 3 rows; the platform (usage ate the budget) caps 'result' at 2."""
475
+ api, mgr = _multi('1', server_max='0.032')
476
+ store = Store()
477
+ r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
478
+ # The platform's remaining budget is only learnt from the short answer, so the
479
+ # retry keeps what that answer proves affordable: fewer rows, never a part-paid one.
480
+ assert 1 <= r.charged_count <= 2
481
+ assert len(store.items) == r.charged_count
482
+ assert api.spent == Decimal('0.016') * r.charged_count
483
+ assert api.voids # the short attempt was voided, not left part-paid
484
+ assert sorted(n for n, _ in api.ops.values()) == ['page', 'result']
485
+
486
+
487
+ @pytest.mark.asyncio
488
+ async def test_multi_event_refusal_stores_nothing_and_bills_nothing() -> None:
489
+ api, mgr = _multi('0.015') # less than one full row
490
+ store = Store()
491
+ r = await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push)
492
+ assert r.charged_count == 0 and r.event_charge_limit_reached is True
493
+ assert store.items == [] and api.spent == Decimal('0')
494
+
495
+
496
+ @pytest.mark.asyncio
497
+ async def test_multi_event_push_does_not_touch_prepaid_balance() -> None:
498
+ api, mgr = _multi('0.026')
499
+ store = Store()
500
+ await mgr.charge('result', 1) # prepays one plain row
501
+ await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push) # $0.016, paid itself
502
+ assert mgr._prepaid_row_balance == 1
503
+ await _plain_push(mgr, store, [{'n': 2}]) # the prepaid row
504
+ assert [i['n'] for i in store.items] == [1, 2]
505
+
506
+
507
+ def test_multi_event_validation() -> None:
508
+ from scrapely.charging import _validate_row_charges
509
+ for bad in ({}, {'result': 0}, {'result': 1.5}, {'apify-default-dataset-item': 1}, {'': 1}, None):
510
+ with pytest.raises(ValueError):
511
+ _validate_row_charges(bad)