scrapely-python 0.1.17__tar.gz → 0.1.19__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapely_python-0.1.17/src/scrapely_python.egg-info → scrapely_python-0.1.19}/PKG-INFO +1 -1
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/pyproject.toml +1 -1
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/__init__.py +1 -1
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/actor.py +16 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/charging.py +213 -5
- {scrapely_python-0.1.17 → scrapely_python-0.1.19/src/scrapely_python.egg-info}/PKG-INFO +1 -1
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_charge_limit_guard.py +185 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/LICENSE +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/LICENSE-APACHE +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/NOTICE +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/README.md +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/setup.cfg +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/_browser_locations.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/browser_configuration.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/cdp_retry.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/charging_client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/configuration.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/crypto/__init__.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/crypto/_secrets.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/errors.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/events/__init__.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/events/_types.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/events/event_manager.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/log.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/patch_browser_pool.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/pricing_types.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/proxy_configuration.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/py.typed +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/request_loaders/__init__.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/request_loaders/_scrapely_request_list.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/solver.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/__init__.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/_file_system/__init__.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/_file_system/_key_value_store_client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/_file_system/_storage_client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/_metadata.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/api_client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/patch_storage.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_dataset_client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_kvs_client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_rq_client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_storage_client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/smart_scrapely_storage_client.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/types.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/utils.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage_clients/__init__.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storages/__init__.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/types.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/webhook.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/SOURCES.txt +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/dependency_links.txt +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/requires.txt +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/top_level.txt +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_api_client_retries.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_apify_parity.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_browser_configuration.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_cdp_retry.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_charge_retries.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_charging_parity.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_charging_pending.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_input_proxy_aliases.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_parity.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_patch_browser_pool.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_proxy_configuration.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_push_idempotency.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_run_control.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_secrets.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_solver.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_storage.py +0 -0
- {scrapely_python-0.1.17 → scrapely_python-0.1.19}/tests/test_terminal_status_message.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scrapely-python"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.19"
|
|
8
8
|
description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT AND Apache-2.0"
|
|
@@ -28,6 +28,7 @@ from .charging import (
|
|
|
28
28
|
ChargingManager,
|
|
29
29
|
is_default_dataset_item_event,
|
|
30
30
|
push_data_and_charge,
|
|
31
|
+
push_data_with_charges,
|
|
31
32
|
set_active_charging_manager,
|
|
32
33
|
)
|
|
33
34
|
from .configuration import Configuration
|
|
@@ -781,6 +782,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
781
782
|
charged_event_name: Optional[str] = None,
|
|
782
783
|
*,
|
|
783
784
|
event_name: Optional[str] = None,
|
|
785
|
+
charges: Optional[Dict[str, int]] = None,
|
|
784
786
|
) -> ChargeResult:
|
|
785
787
|
"""Push data to the default dataset, charging pay-per-event items.
|
|
786
788
|
|
|
@@ -796,6 +798,10 @@ class Actor(metaclass=_ActorMeta):
|
|
|
796
798
|
(Apify Python SDK name; also accepted positionally).
|
|
797
799
|
event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
|
|
798
800
|
kept so existing callers keep working.
|
|
801
|
+
charges: Several events per row, charged atomically: ``{'result': 1, 'page': 3}``
|
|
802
|
+
charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
|
|
803
|
+
only when all of its events were charged; rows the spending limit cannot
|
|
804
|
+
pay for in full are not stored. Cannot be combined with ``charged_event_name``.
|
|
799
805
|
|
|
800
806
|
Returns:
|
|
801
807
|
ChargeResult describing how many items were charged and pushed.
|
|
@@ -818,6 +824,16 @@ class Actor(metaclass=_ActorMeta):
|
|
|
818
824
|
|
|
819
825
|
items = data if isinstance(data, list) else [data]
|
|
820
826
|
charging_manager = cls.get_charging_manager()
|
|
827
|
+
if charges is not None:
|
|
828
|
+
if event_name is not None:
|
|
829
|
+
raise ValueError('Pass either charged_event_name or charges, not both')
|
|
830
|
+
return await push_data_with_charges(
|
|
831
|
+
charging_manager,
|
|
832
|
+
items,
|
|
833
|
+
charges,
|
|
834
|
+
True, # Actor.push_data always targets the default dataset
|
|
835
|
+
cls._default_dataset.push_data,
|
|
836
|
+
)
|
|
821
837
|
return await push_data_and_charge(
|
|
822
838
|
charging_manager,
|
|
823
839
|
items,
|
|
@@ -240,6 +240,12 @@ class ChargingManager:
|
|
|
240
240
|
self._charged_events: Dict[str, int] = {}
|
|
241
241
|
self._max_total_charge_usd: Optional[Decimal] = None
|
|
242
242
|
self._total_charged_usd: Decimal = Decimal('0.0')
|
|
243
|
+
# Run spend the platform counts against maxTotalChargeUsd that this SDK
|
|
244
|
+
# does not charge itself: compute/proxy/browser usage and server-billed
|
|
245
|
+
# events. Learnt from the charge endpoint's remainingBudgetNd (only ever
|
|
246
|
+
# grows), so the local budget gates on the run's true remaining budget.
|
|
247
|
+
self._platform_other_usd: Decimal = Decimal('0')
|
|
248
|
+
self._last_remaining_budget_nd: Optional[int] = None
|
|
243
249
|
self._charge_lock = RLock()
|
|
244
250
|
self._flush_lock = asyncio.Lock()
|
|
245
251
|
# Serializes budget calculation/reservation with PPE-aware dataset pushes.
|
|
@@ -288,6 +294,27 @@ class ChargingManager:
|
|
|
288
294
|
self._refused_default_dataset_items,
|
|
289
295
|
)
|
|
290
296
|
|
|
297
|
+
def _remaining_budget_usd(self) -> Decimal:
|
|
298
|
+
"""maxTotalChargeUsd minus this SDK's charges and the platform's other run spend."""
|
|
299
|
+
assert self._max_total_charge_usd is not None
|
|
300
|
+
return self._max_total_charge_usd - self._total_charged_usd - self._platform_other_usd
|
|
301
|
+
|
|
302
|
+
def _sync_platform_budget(self) -> None:
|
|
303
|
+
"""Fold the last charge response's remainingBudgetNd into the local budget.
|
|
304
|
+
|
|
305
|
+
Called after the local reservation matches what the platform accepted.
|
|
306
|
+
The platform's remaining budget = max - (our charges + usage + server
|
|
307
|
+
events), so ``other`` = max - remaining - our charges. Monotonic: usage
|
|
308
|
+
never shrinks, and a stale answer must not hand budget back.
|
|
309
|
+
"""
|
|
310
|
+
nd = self._last_remaining_budget_nd
|
|
311
|
+
self._last_remaining_budget_nd = None
|
|
312
|
+
if nd is None or self._max_total_charge_usd is None or nd < 0:
|
|
313
|
+
return
|
|
314
|
+
other = self._max_total_charge_usd - Decimal(nd) / Decimal(1_000_000_000) - self._total_charged_usd
|
|
315
|
+
if other > self._platform_other_usd:
|
|
316
|
+
self._platform_other_usd = other
|
|
317
|
+
|
|
291
318
|
def is_charge_limit_reached(self) -> bool:
|
|
292
319
|
"""Whether this pay-per-event run has used up its spending limit.
|
|
293
320
|
|
|
@@ -307,7 +334,7 @@ class ChargingManager:
|
|
|
307
334
|
prices = [price for price in self._per_event_prices.values() if price > 0]
|
|
308
335
|
if not prices:
|
|
309
336
|
return False
|
|
310
|
-
return self.
|
|
337
|
+
return self._remaining_budget_usd() < min(prices)
|
|
311
338
|
|
|
312
339
|
def _mark_charge_limit_hit(self) -> None:
|
|
313
340
|
if self._pricing_info is not None:
|
|
@@ -795,6 +822,7 @@ class ChargingManager:
|
|
|
795
822
|
idempotency_key=idempotency_key,
|
|
796
823
|
deadline=deadline,
|
|
797
824
|
)
|
|
825
|
+
self._last_remaining_budget_nd = _read_remaining_budget_nd(body)
|
|
798
826
|
return _read_accepted_count(body, count)
|
|
799
827
|
except Exception as e:
|
|
800
828
|
last_error = e
|
|
@@ -957,7 +985,7 @@ class ChargingManager:
|
|
|
957
985
|
# legacy one-event overdraw while the budget is not yet exceeded.
|
|
958
986
|
chargeable_count = count
|
|
959
987
|
if self._max_total_charge_usd is not None and affordability_price > 0:
|
|
960
|
-
remaining = self.
|
|
988
|
+
remaining = self._remaining_budget_usd()
|
|
961
989
|
max_count_by_limit = max(0, int(remaining / affordability_price))
|
|
962
990
|
if count > max_count_by_limit:
|
|
963
991
|
chargeable_count = (
|
|
@@ -1045,6 +1073,15 @@ class ChargingManager:
|
|
|
1045
1073
|
charged_count=chargeable_count,
|
|
1046
1074
|
event_charge_limit_reached=self._is_event_limit_reached(event_name),
|
|
1047
1075
|
)
|
|
1076
|
+
self._sync_platform_budget()
|
|
1077
|
+
if self._max_total_charge_usd is not None and self._platform_other_usd > 0:
|
|
1078
|
+
# Usage may have used up what is left: report it on this result.
|
|
1079
|
+
result = ChargeResult(
|
|
1080
|
+
chargeable_within_limit=self._compute_chargeable(),
|
|
1081
|
+
charged_count=result.charged_count,
|
|
1082
|
+
event_charge_limit_reached=result.event_charge_limit_reached
|
|
1083
|
+
or self._is_event_limit_reached(event_name),
|
|
1084
|
+
)
|
|
1048
1085
|
else:
|
|
1049
1086
|
raise ChargeUncertainError(
|
|
1050
1087
|
f'Charge for event {event_name!r} (count={chargeable_count}) '
|
|
@@ -1103,7 +1140,7 @@ class ChargingManager:
|
|
|
1103
1140
|
if event_price is None or event_price <= 0:
|
|
1104
1141
|
return None
|
|
1105
1142
|
|
|
1106
|
-
remaining = self.
|
|
1143
|
+
remaining = self._remaining_budget_usd()
|
|
1107
1144
|
return max(0, int(remaining / event_price))
|
|
1108
1145
|
|
|
1109
1146
|
def calculate_total_charged_amount(self) -> Decimal:
|
|
@@ -1165,7 +1202,7 @@ class ChargingManager:
|
|
|
1165
1202
|
if self._max_total_charge_usd is None:
|
|
1166
1203
|
return items_count
|
|
1167
1204
|
|
|
1168
|
-
remaining = self.
|
|
1205
|
+
remaining = self._remaining_budget_usd()
|
|
1169
1206
|
max_items = int(remaining / combined_price)
|
|
1170
1207
|
|
|
1171
1208
|
return min(items_count, max(0, max_items))
|
|
@@ -1226,7 +1263,7 @@ class ChargingManager:
|
|
|
1226
1263
|
if self._max_total_charge_usd is None:
|
|
1227
1264
|
return False
|
|
1228
1265
|
|
|
1229
|
-
remaining = self.
|
|
1266
|
+
remaining = self._remaining_budget_usd()
|
|
1230
1267
|
return remaining < event_price
|
|
1231
1268
|
|
|
1232
1269
|
def _compute_chargeable(self) -> Dict[str, Optional[int]]:
|
|
@@ -1264,6 +1301,17 @@ def _is_charge_limit_rejection(err: ChargeRejectedError) -> bool:
|
|
|
1264
1301
|
)
|
|
1265
1302
|
|
|
1266
1303
|
|
|
1304
|
+
def _read_remaining_budget_nd(body: Any) -> Optional[int]:
|
|
1305
|
+
"""remainingBudgetNd from a charge response (platforms that send it), else None."""
|
|
1306
|
+
if not isinstance(body, dict):
|
|
1307
|
+
return None
|
|
1308
|
+
data = body.get('data') if isinstance(body.get('data'), dict) else body
|
|
1309
|
+
raw = data.get('remainingBudgetNd') if isinstance(data, dict) else None
|
|
1310
|
+
if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw >= 0:
|
|
1311
|
+
return int(raw)
|
|
1312
|
+
return None
|
|
1313
|
+
|
|
1314
|
+
|
|
1267
1315
|
def _read_accepted_count(body: Any, fallback: int) -> int:
|
|
1268
1316
|
if not isinstance(body, dict):
|
|
1269
1317
|
return fallback
|
|
@@ -1585,6 +1633,166 @@ async def push_data_and_charge(
|
|
|
1585
1633
|
)
|
|
1586
1634
|
|
|
1587
1635
|
|
|
1636
|
+
def _validate_row_charges(charges: Any) -> Dict[str, int]:
|
|
1637
|
+
if not isinstance(charges, dict) or not charges:
|
|
1638
|
+
raise ValueError('charges must be a non-empty dict of {event_name: count_per_row}')
|
|
1639
|
+
out: Dict[str, int] = {}
|
|
1640
|
+
for name, count in charges.items():
|
|
1641
|
+
if not isinstance(name, str) or not name:
|
|
1642
|
+
raise ValueError('charges keys must be event names')
|
|
1643
|
+
if is_default_dataset_item_event(name):
|
|
1644
|
+
raise ValueError(f"Cannot charge for synthetic event '{name}' manually")
|
|
1645
|
+
if not isinstance(count, int) or isinstance(count, bool) or count < 1:
|
|
1646
|
+
raise ValueError(f'charges[{name!r}] must be a positive integer (events per row)')
|
|
1647
|
+
out[name] = count
|
|
1648
|
+
return out
|
|
1649
|
+
|
|
1650
|
+
|
|
1651
|
+
async def push_data_with_charges(
|
|
1652
|
+
charging_manager: ChargingManager,
|
|
1653
|
+
items: Union[Dict[str, Any], List[Dict[str, Any]]],
|
|
1654
|
+
charges: Dict[str, int],
|
|
1655
|
+
is_default_dataset: bool,
|
|
1656
|
+
push_fn: Any,
|
|
1657
|
+
) -> ChargeResult:
|
|
1658
|
+
"""Push rows that each cost several events, atomically per row.
|
|
1659
|
+
|
|
1660
|
+
``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
|
|
1661
|
+
A row is stored only when every one of its events was charged: the full set
|
|
1662
|
+
is charged first, rows the budget cannot fully pay for are not stored, and
|
|
1663
|
+
any operation that came back short is voided and re-charged for the rows
|
|
1664
|
+
that are kept, so no row is ever stored partly paid and no event is billed
|
|
1665
|
+
for a row that was not stored. Unpriced events are ignored. On the default
|
|
1666
|
+
dataset the default-dataset-item event (when priced) is added per row.
|
|
1667
|
+
"""
|
|
1668
|
+
charges = _validate_row_charges(charges)
|
|
1669
|
+
rows_in = items if isinstance(items, list) else [items]
|
|
1670
|
+
if not rows_in:
|
|
1671
|
+
return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
|
|
1672
|
+
|
|
1673
|
+
async with charging_manager._operation_lock:
|
|
1674
|
+
if charging_manager._closed:
|
|
1675
|
+
raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
|
|
1676
|
+
voids_confirmed = await charging_manager._retry_pending_voids()
|
|
1677
|
+
|
|
1678
|
+
if charging_manager._pricing_info is None:
|
|
1679
|
+
await _gated_push(push_fn, rows_in)
|
|
1680
|
+
return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
|
|
1681
|
+
|
|
1682
|
+
per_row: Dict[str, int] = {
|
|
1683
|
+
name: n for name, n in charges.items()
|
|
1684
|
+
if (charging_manager._calculate_event_price(name) or Decimal('0')) > 0
|
|
1685
|
+
}
|
|
1686
|
+
if is_default_dataset:
|
|
1687
|
+
default_event = charging_manager._config.get('default_dataset_item_event_name') or \
|
|
1688
|
+
resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
|
|
1689
|
+
if default_event and default_event not in per_row and \
|
|
1690
|
+
(charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
|
|
1691
|
+
per_row[default_event] = 1
|
|
1692
|
+
if not per_row:
|
|
1693
|
+
# Every listed event is free: a plain (unpaid) push.
|
|
1694
|
+
return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
|
|
1695
|
+
if not voids_confirmed and any(not is_default_dataset_item_event(n) for n in per_row):
|
|
1696
|
+
raise RuntimeError('An earlier charge for a failed dataset push could not be voided yet; retry the push')
|
|
1697
|
+
|
|
1698
|
+
# Rows the budget can pay for in full (no one-event overdraw: a row is all or nothing).
|
|
1699
|
+
row_cost = sum(
|
|
1700
|
+
(charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
|
|
1701
|
+
)
|
|
1702
|
+
rows = len(rows_in)
|
|
1703
|
+
if charging_manager._max_total_charge_usd is not None and row_cost > 0:
|
|
1704
|
+
remaining = charging_manager._remaining_budget_usd()
|
|
1705
|
+
rows = min(rows, max(0, int(remaining / row_cost)))
|
|
1706
|
+
|
|
1707
|
+
ops: List[tuple] = [] # (event, charged, op_id, uncertain)
|
|
1708
|
+
|
|
1709
|
+
async def undo(entries: List[tuple]) -> None:
|
|
1710
|
+
for name, charged, op_id, uncertain in entries:
|
|
1711
|
+
if charged <= 0:
|
|
1712
|
+
continue
|
|
1713
|
+
if op_id is None:
|
|
1714
|
+
charging_manager._unwind_reservation(name, charged)
|
|
1715
|
+
elif uncertain:
|
|
1716
|
+
await charging_manager._cancel_uncertain_charge(name, op_id, charged)
|
|
1717
|
+
else:
|
|
1718
|
+
await charging_manager._void_charge(name, op_id, charged)
|
|
1719
|
+
|
|
1720
|
+
for _attempt in range(4):
|
|
1721
|
+
if rows <= 0:
|
|
1722
|
+
break
|
|
1723
|
+
ops = []
|
|
1724
|
+
short_to: Optional[int] = None
|
|
1725
|
+
for name, n in sorted(per_row.items(), key=lambda kv: (is_default_dataset_item_event(kv[0]), kv[0])):
|
|
1726
|
+
want = n * rows
|
|
1727
|
+
charging_manager._take_last_accepted_operation_id()
|
|
1728
|
+
try:
|
|
1729
|
+
r = await charging_manager._charge_exclusive(name, want)
|
|
1730
|
+
op_id = charging_manager._take_last_accepted_operation_id()
|
|
1731
|
+
ops.append((name, r.charged_count, op_id, False))
|
|
1732
|
+
got = r.charged_count
|
|
1733
|
+
except ChargeUncertainError as unconfirmed:
|
|
1734
|
+
# Not confirmed: the intent stays pending and is re-posted at exit,
|
|
1735
|
+
# so it counts as accepted (items stored, billed once).
|
|
1736
|
+
ops.append((name, unconfirmed.count, unconfirmed.operation_id, True))
|
|
1737
|
+
got = unconfirmed.count
|
|
1738
|
+
if got < want:
|
|
1739
|
+
short_to = got // n
|
|
1740
|
+
break
|
|
1741
|
+
if short_to is None:
|
|
1742
|
+
break
|
|
1743
|
+
# Some event came back short: void this attempt in full and retry with
|
|
1744
|
+
# the rows that event can pay for (a void restores the budget).
|
|
1745
|
+
await undo(ops)
|
|
1746
|
+
ops = []
|
|
1747
|
+
rows = short_to # got < n * rows, so short_to < rows
|
|
1748
|
+
|
|
1749
|
+
if rows <= 0 or not ops:
|
|
1750
|
+
await undo(ops)
|
|
1751
|
+
charging_manager._mark_charge_limit_hit()
|
|
1752
|
+
if is_default_dataset:
|
|
1753
|
+
charging_manager._note_refused_default_dataset_items(len(rows_in))
|
|
1754
|
+
return ChargeResult(
|
|
1755
|
+
chargeable_within_limit=charging_manager.compute_chargeable(),
|
|
1756
|
+
charged_count=0,
|
|
1757
|
+
event_charge_limit_reached=True,
|
|
1758
|
+
)
|
|
1759
|
+
|
|
1760
|
+
kept = rows_in[:rows]
|
|
1761
|
+
if rows < len(rows_in):
|
|
1762
|
+
charging_manager._mark_charge_limit_hit()
|
|
1763
|
+
if is_default_dataset:
|
|
1764
|
+
charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
|
|
1765
|
+
try:
|
|
1766
|
+
await _gated_push(push_fn, kept)
|
|
1767
|
+
except Exception as push_err:
|
|
1768
|
+
logger.error('Dataset push failed after charges accepted (rows=%s): %s; voiding the charges', rows, push_err)
|
|
1769
|
+
if _push_certainly_failed(push_err):
|
|
1770
|
+
await undo(ops)
|
|
1771
|
+
raise
|
|
1772
|
+
return ChargeResult(
|
|
1773
|
+
chargeable_within_limit=charging_manager.compute_chargeable(),
|
|
1774
|
+
charged_count=rows,
|
|
1775
|
+
event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
|
|
1776
|
+
)
|
|
1777
|
+
|
|
1778
|
+
|
|
1779
|
+
async def _unpaid_push_locked(
|
|
1780
|
+
charging_manager: ChargingManager, rows: List[Dict[str, Any]], is_default_dataset: bool, push_fn: Any
|
|
1781
|
+
) -> ChargeResult:
|
|
1782
|
+
"""A push whose rows carry no priced event (caller holds the operation lock)."""
|
|
1783
|
+
past_limit = charging_manager.is_charge_limit_reached()
|
|
1784
|
+
allowed = charging_manager._take_unpaid_rows(len(rows)) if is_default_dataset else len(rows)
|
|
1785
|
+
if allowed < len(rows):
|
|
1786
|
+
charging_manager._note_refused_default_dataset_items(len(rows) - allowed)
|
|
1787
|
+
if allowed > 0:
|
|
1788
|
+
await _gated_push(push_fn, rows[:allowed])
|
|
1789
|
+
return ChargeResult(
|
|
1790
|
+
chargeable_within_limit=charging_manager.compute_chargeable(),
|
|
1791
|
+
charged_count=0,
|
|
1792
|
+
event_charge_limit_reached=past_limit,
|
|
1793
|
+
)
|
|
1794
|
+
|
|
1795
|
+
|
|
1588
1796
|
def _push_is_paid_per_item(
|
|
1589
1797
|
charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
|
|
1590
1798
|
) -> bool:
|
|
@@ -400,3 +400,188 @@ async def test_refused_charge_402_then_plain_push_is_not_saved() -> None:
|
|
|
400
400
|
await _plain_push(mgr, store, [{'n': n}])
|
|
401
401
|
|
|
402
402
|
assert [i['n'] for i in store.items] == [0, 1]
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
# ---- atomic multi-event push: push_data(row, charges={...}) ----
|
|
406
|
+
|
|
407
|
+
from scrapely.charging import push_data_with_charges # noqa: E402
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
class MultiApi(FakeChargeApi):
|
|
411
|
+
"""Charge endpoint priced per event, with void support (by Idempotency-Key)."""
|
|
412
|
+
|
|
413
|
+
def __init__(self, prices: Dict[str, Decimal], max_usd: Optional[Decimal]) -> None:
|
|
414
|
+
super().__init__(price=Decimal('0'), max_usd=max_usd)
|
|
415
|
+
self.prices = prices
|
|
416
|
+
self.ops: Dict[str, tuple] = {}
|
|
417
|
+
self.voids: List[str] = []
|
|
418
|
+
|
|
419
|
+
async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **_: Any) -> Dict[str, Any]:
|
|
420
|
+
key = (headers or {}).get('Idempotency-Key', '')
|
|
421
|
+
if path.endswith('/charge/void'):
|
|
422
|
+
name, accepted = self.ops.pop(key)
|
|
423
|
+
self.spent -= self.prices[name] * accepted
|
|
424
|
+
self.voids.append(name)
|
|
425
|
+
return {}
|
|
426
|
+
name = (json_data or {}).get('eventName')
|
|
427
|
+
count = int((json_data or {}).get('count', 1))
|
|
428
|
+
price = self.prices[name]
|
|
429
|
+
accepted = count
|
|
430
|
+
if self.max_usd is not None and price > 0:
|
|
431
|
+
affordable = int((self.max_usd - self.spent) / price)
|
|
432
|
+
if affordable < 1:
|
|
433
|
+
raise HttpError(402, 'charge-limit-exceeded')
|
|
434
|
+
accepted = min(count, affordable)
|
|
435
|
+
self.spent += price * accepted
|
|
436
|
+
self.ops[key] = (name, accepted)
|
|
437
|
+
return {'acceptedCount': accepted, 'requestedCount': count}
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
PRICES = {'result': Decimal('0.01'), 'page': Decimal('0.002')} # one row = 1 result + 3 pages = $0.016
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _multi(max_usd: str, server_max: Optional[str] = None):
|
|
444
|
+
api = MultiApi(PRICES, Decimal(server_max or max_usd))
|
|
445
|
+
return api, _manager(api, prices=dict(PRICES), max_usd=max_usd)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
@pytest.mark.asyncio
|
|
449
|
+
async def test_multi_event_exact_budget_stores_every_paid_row() -> None:
|
|
450
|
+
api, mgr = _multi('0.048') # exactly 3 rows
|
|
451
|
+
store = Store()
|
|
452
|
+
r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
|
|
453
|
+
assert r.charged_count == 3
|
|
454
|
+
assert [i['n'] for i in store.items] == [0, 1, 2]
|
|
455
|
+
assert api.spent == Decimal('0.048')
|
|
456
|
+
assert mgr.is_charge_limit_reached() is True
|
|
457
|
+
again = await push_data_with_charges(mgr, [{'n': 9}], {'result': 1, 'page': 3}, True, store.push)
|
|
458
|
+
assert again.charged_count == 0 and again.event_charge_limit_reached is True
|
|
459
|
+
assert len(store.items) == 3 and api.spent == Decimal('0.048')
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
@pytest.mark.asyncio
|
|
463
|
+
async def test_multi_event_partial_budget_stores_only_fully_paid_rows() -> None:
|
|
464
|
+
api, mgr = _multi('0.040') # 2.5 rows: only 2 fully paid
|
|
465
|
+
store = Store()
|
|
466
|
+
r = await push_data_with_charges(mgr, [{'n': n} for n in range(5)], {'result': 1, 'page': 3}, True, store.push)
|
|
467
|
+
assert r.charged_count == 2 and r.event_charge_limit_reached is True
|
|
468
|
+
assert [i['n'] for i in store.items] == [0, 1]
|
|
469
|
+
assert api.spent == Decimal('0.032') # never a part-paid row
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
@pytest.mark.asyncio
|
|
473
|
+
async def test_multi_event_server_cap_short_voids_and_recharges() -> None:
|
|
474
|
+
"""Local budget says 3 rows; the platform (usage ate the budget) caps 'result' at 2."""
|
|
475
|
+
api, mgr = _multi('1', server_max='0.032')
|
|
476
|
+
store = Store()
|
|
477
|
+
r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
|
|
478
|
+
# The platform's remaining budget is only learnt from the short answer, so the
|
|
479
|
+
# retry keeps what that answer proves affordable: fewer rows, never a part-paid one.
|
|
480
|
+
assert 1 <= r.charged_count <= 2
|
|
481
|
+
assert len(store.items) == r.charged_count
|
|
482
|
+
assert api.spent == Decimal('0.016') * r.charged_count
|
|
483
|
+
assert api.voids # the short attempt was voided, not left part-paid
|
|
484
|
+
assert sorted(n for n, _ in api.ops.values()) == ['page', 'result']
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
@pytest.mark.asyncio
|
|
488
|
+
async def test_multi_event_refusal_stores_nothing_and_bills_nothing() -> None:
|
|
489
|
+
api, mgr = _multi('0.015') # less than one full row
|
|
490
|
+
store = Store()
|
|
491
|
+
r = await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push)
|
|
492
|
+
assert r.charged_count == 0 and r.event_charge_limit_reached is True
|
|
493
|
+
assert store.items == [] and api.spent == Decimal('0')
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
@pytest.mark.asyncio
|
|
497
|
+
async def test_multi_event_push_does_not_touch_prepaid_balance() -> None:
|
|
498
|
+
api, mgr = _multi('0.026')
|
|
499
|
+
store = Store()
|
|
500
|
+
await mgr.charge('result', 1) # prepays one plain row
|
|
501
|
+
await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push) # $0.016, paid itself
|
|
502
|
+
assert mgr._prepaid_row_balance == 1
|
|
503
|
+
await _plain_push(mgr, store, [{'n': 2}]) # the prepaid row
|
|
504
|
+
assert [i['n'] for i in store.items] == [1, 2]
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def test_multi_event_validation() -> None:
|
|
508
|
+
from scrapely.charging import _validate_row_charges
|
|
509
|
+
for bad in ({}, {'result': 0}, {'result': 1.5}, {'apify-default-dataset-item': 1}, {'': 1}, None):
|
|
510
|
+
with pytest.raises(ValueError):
|
|
511
|
+
_validate_row_charges(bad)
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
# ---- platform usage counts against the cap (remainingBudgetNd in charge responses) ----
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
class UsageApi(FakeChargeApi):
|
|
518
|
+
"""Ledger whose cap also covers platform usage, reporting remainingBudgetNd like the server."""
|
|
519
|
+
|
|
520
|
+
def __init__(self, *, price: Decimal, max_usd: Decimal, usage: Decimal, send_remaining: bool = True) -> None:
|
|
521
|
+
super().__init__(price=price, max_usd=max_usd)
|
|
522
|
+
self.usage = usage
|
|
523
|
+
self.send_remaining = send_remaining
|
|
524
|
+
|
|
525
|
+
async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, **_: Any) -> Dict[str, Any]:
|
|
526
|
+
count = int((json_data or {}).get('count', 1))
|
|
527
|
+
left = self.max_usd - self.spent - self.usage
|
|
528
|
+
affordable = int(left / self.price)
|
|
529
|
+
if affordable < 1:
|
|
530
|
+
raise HttpError(402, 'charge-limit-exceeded')
|
|
531
|
+
accepted = min(count, affordable)
|
|
532
|
+
self.spent += self.price * accepted
|
|
533
|
+
body: Dict[str, Any] = {'acceptedCount': accepted, 'requestedCount': count}
|
|
534
|
+
if self.send_remaining:
|
|
535
|
+
body['remainingBudgetNd'] = int((self.max_usd - self.spent - self.usage) * 1_000_000_000)
|
|
536
|
+
return body
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
@pytest.mark.asyncio
|
|
540
|
+
async def test_platform_usage_shrinks_the_local_budget() -> None:
|
|
541
|
+
api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('0.05'), usage=Decimal('0.03'))
|
|
542
|
+
mgr = _manager(api, max_usd='0.05')
|
|
543
|
+
store = Store()
|
|
544
|
+
|
|
545
|
+
r = await mgr.charge('result', 1)
|
|
546
|
+
assert r.charged_count == 1
|
|
547
|
+
# $0.05 cap - $0.01 charged - $0.03 usage: one more event fits, not four.
|
|
548
|
+
assert mgr.calculate_max_event_charge_count_within_limit('result') == 1
|
|
549
|
+
assert mgr.is_charge_limit_reached() is False
|
|
550
|
+
pushed = await _charged_push(mgr, store, [{'n': n} for n in range(5)])
|
|
551
|
+
assert pushed.charged_count == 1 and pushed.event_charge_limit_reached is True
|
|
552
|
+
assert len(store.items) == 1
|
|
553
|
+
assert mgr.is_charge_limit_reached() is True
|
|
554
|
+
# The prepaid row from the first charge is still saved; nothing beyond it.
|
|
555
|
+
await _plain_push(mgr, store, [{'p': 1}, {'p': 2}])
|
|
556
|
+
assert len(store.items) == 2
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
@pytest.mark.asyncio
|
|
560
|
+
async def test_usage_learnt_from_a_charge_marks_the_limit_on_that_result() -> None:
|
|
561
|
+
api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('0.05'), usage=Decimal('0.04'))
|
|
562
|
+
mgr = _manager(api, max_usd='0.05')
|
|
563
|
+
r = await mgr.charge('result', 1)
|
|
564
|
+
assert r.charged_count == 1
|
|
565
|
+
assert r.event_charge_limit_reached is True
|
|
566
|
+
assert mgr.is_charge_limit_reached() is True
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
@pytest.mark.asyncio
|
|
570
|
+
async def test_without_remaining_budget_in_responses_behaviour_is_unchanged() -> None:
|
|
571
|
+
api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('0.05'), usage=Decimal('0.03'), send_remaining=False)
|
|
572
|
+
mgr = _manager(api, max_usd='0.05')
|
|
573
|
+
await mgr.charge('result', 1)
|
|
574
|
+
assert mgr.calculate_max_event_charge_count_within_limit('result') == 4
|
|
575
|
+
assert mgr._platform_other_usd == 0
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
@pytest.mark.asyncio
|
|
579
|
+
async def test_platform_other_spend_never_shrinks() -> None:
|
|
580
|
+
api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('1'), usage=Decimal('0.30'))
|
|
581
|
+
mgr = _manager(api, max_usd='1')
|
|
582
|
+
await mgr.charge('result', 1)
|
|
583
|
+
seen = mgr._platform_other_usd
|
|
584
|
+
assert seen == Decimal('0.30')
|
|
585
|
+
api.usage = Decimal('0.10') # a stale or lower report
|
|
586
|
+
await mgr.charge('result', 1)
|
|
587
|
+
assert mgr._platform_other_usd == seen
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/_file_system/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_dataset_client.py
RENAMED
|
File without changes
|
{scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_kvs_client.py
RENAMED
|
File without changes
|
{scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_rq_client.py
RENAMED
|
File without changes
|
{scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely/storage/scrapely_storage_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
{scrapely_python-0.1.17 → scrapely_python-0.1.19}/src/scrapely_python.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|