scrapely-python 0.1.16__tar.gz → 0.1.18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapely_python-0.1.16/src/scrapely_python.egg-info → scrapely_python-0.1.18}/PKG-INFO +1 -1
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/pyproject.toml +1 -1
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/__init__.py +1 -1
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/actor.py +16 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/charging.py +221 -11
- {scrapely_python-0.1.16 → scrapely_python-0.1.18/src/scrapely_python.egg-info}/PKG-INFO +1 -1
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_charge_limit_guard.py +220 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/LICENSE +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/LICENSE-APACHE +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/NOTICE +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/README.md +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/setup.cfg +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/_browser_locations.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/browser_configuration.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/cdp_retry.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/charging_client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/configuration.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/crypto/__init__.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/crypto/_secrets.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/errors.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/events/__init__.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/events/_types.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/events/event_manager.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/log.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/patch_browser_pool.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/pricing_types.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/proxy_configuration.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/py.typed +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/request_loaders/__init__.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/request_loaders/_scrapely_request_list.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/solver.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/__init__.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/_file_system/__init__.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/_file_system/_key_value_store_client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/_file_system/_storage_client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/_metadata.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/api_client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/patch_storage.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_dataset_client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_kvs_client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_rq_client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_storage_client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/smart_scrapely_storage_client.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/types.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/utils.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage_clients/__init__.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storages/__init__.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/types.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/webhook.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/SOURCES.txt +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/dependency_links.txt +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/requires.txt +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/top_level.txt +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_api_client_retries.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_apify_parity.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_browser_configuration.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_cdp_retry.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_charge_retries.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_charging_parity.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_charging_pending.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_input_proxy_aliases.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_parity.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_patch_browser_pool.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_proxy_configuration.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_push_idempotency.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_run_control.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_secrets.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_solver.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_storage.py +0 -0
- {scrapely_python-0.1.16 → scrapely_python-0.1.18}/tests/test_terminal_status_message.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scrapely-python"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.18"
|
|
8
8
|
description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT AND Apache-2.0"
|
|
@@ -28,6 +28,7 @@ from .charging import (
|
|
|
28
28
|
ChargingManager,
|
|
29
29
|
is_default_dataset_item_event,
|
|
30
30
|
push_data_and_charge,
|
|
31
|
+
push_data_with_charges,
|
|
31
32
|
set_active_charging_manager,
|
|
32
33
|
)
|
|
33
34
|
from .configuration import Configuration
|
|
@@ -781,6 +782,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
781
782
|
charged_event_name: Optional[str] = None,
|
|
782
783
|
*,
|
|
783
784
|
event_name: Optional[str] = None,
|
|
785
|
+
charges: Optional[Dict[str, int]] = None,
|
|
784
786
|
) -> ChargeResult:
|
|
785
787
|
"""Push data to the default dataset, charging pay-per-event items.
|
|
786
788
|
|
|
@@ -796,6 +798,10 @@ class Actor(metaclass=_ActorMeta):
|
|
|
796
798
|
(Apify Python SDK name; also accepted positionally).
|
|
797
799
|
event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
|
|
798
800
|
kept so existing callers keep working.
|
|
801
|
+
charges: Several events per row, charged atomically: ``{'result': 1, 'page': 3}``
|
|
802
|
+
charges 1 ``result`` and 3 ``page`` events for every row. A row is stored
|
|
803
|
+
only when all of its events were charged; rows the spending limit cannot
|
|
804
|
+
pay for in full are not stored. Cannot be combined with ``charged_event_name``.
|
|
799
805
|
|
|
800
806
|
Returns:
|
|
801
807
|
ChargeResult describing how many items were charged and pushed.
|
|
@@ -818,6 +824,16 @@ class Actor(metaclass=_ActorMeta):
|
|
|
818
824
|
|
|
819
825
|
items = data if isinstance(data, list) else [data]
|
|
820
826
|
charging_manager = cls.get_charging_manager()
|
|
827
|
+
if charges is not None:
|
|
828
|
+
if event_name is not None:
|
|
829
|
+
raise ValueError('Pass either charged_event_name or charges, not both')
|
|
830
|
+
return await push_data_with_charges(
|
|
831
|
+
charging_manager,
|
|
832
|
+
items,
|
|
833
|
+
charges,
|
|
834
|
+
True, # Actor.push_data always targets the default dataset
|
|
835
|
+
cls._default_dataset.push_data,
|
|
836
|
+
)
|
|
821
837
|
return await push_data_and_charge(
|
|
822
838
|
charging_manager,
|
|
823
839
|
items,
|
|
@@ -270,6 +270,13 @@ class ChargingManager:
|
|
|
270
270
|
self._charge_limit_hit = False
|
|
271
271
|
# Default-dataset rows refused after the limit (one warning, then a count).
|
|
272
272
|
self._refused_default_dataset_items = 0
|
|
273
|
+
# Prepaid-row balance: priced events charged through charge() minus rows
|
|
274
|
+
# pushed to the default dataset without a per-row charge. An Actor that
|
|
275
|
+
# charges first and then pushes plainly (charge('product'); push_data(row))
|
|
276
|
+
# has a positive balance after the charge, so the row it already paid for
|
|
277
|
+
# is still saved when that charge used up the spending limit. Push-then-
|
|
278
|
+
# charge Actors never go positive, so they get nothing extra for free.
|
|
279
|
+
self._prepaid_row_balance = 0
|
|
273
280
|
|
|
274
281
|
def close(self) -> None:
|
|
275
282
|
"""Mark the manager closed; subsequent charge/push operations are rejected."""
|
|
@@ -323,8 +330,40 @@ class ChargingManager:
|
|
|
323
330
|
)
|
|
324
331
|
self._refused_default_dataset_items += count
|
|
325
332
|
|
|
333
|
+
def _note_prepaid_charge(self, event_name: str, charged_count: int) -> None:
|
|
334
|
+
if (
|
|
335
|
+
charged_count > 0
|
|
336
|
+
and not is_default_dataset_item_event(event_name)
|
|
337
|
+
and self._per_event_prices.get(event_name, 0) > 0
|
|
338
|
+
):
|
|
339
|
+
self._prepaid_row_balance += charged_count
|
|
340
|
+
|
|
341
|
+
def _take_unpaid_rows(self, item_count: int) -> int:
|
|
342
|
+
"""Account for ``item_count`` rows pushed without a per-row charge.
|
|
343
|
+
|
|
344
|
+
Returns how many of them may be stored: all of them while the limit is
|
|
345
|
+
not reached, afterwards only the rows already paid for by earlier
|
|
346
|
+
charges (the positive prepaid balance).
|
|
347
|
+
"""
|
|
348
|
+
if item_count <= 0:
|
|
349
|
+
return 0
|
|
350
|
+
if not self.is_charge_limit_reached():
|
|
351
|
+
self._prepaid_row_balance -= item_count
|
|
352
|
+
return item_count
|
|
353
|
+
allowed = max(0, min(item_count, self._prepaid_row_balance))
|
|
354
|
+
self._prepaid_row_balance -= allowed
|
|
355
|
+
return allowed
|
|
356
|
+
|
|
326
357
|
def _refuse_default_dataset_push(self, item_count: int) -> bool:
|
|
327
|
-
if item_count <= 0 or
|
|
358
|
+
if item_count <= 0 or self._pricing_info is None:
|
|
359
|
+
return False
|
|
360
|
+
if not self.is_charge_limit_reached():
|
|
361
|
+
self._take_unpaid_rows(item_count)
|
|
362
|
+
return False
|
|
363
|
+
# All-or-nothing here (storage clients cannot store a slice): store the
|
|
364
|
+
# batch only when earlier charges paid for every row of it.
|
|
365
|
+
if self._prepaid_row_balance >= item_count:
|
|
366
|
+
self._prepaid_row_balance -= item_count
|
|
328
367
|
return False
|
|
329
368
|
self._note_refused_default_dataset_items(item_count)
|
|
330
369
|
return True
|
|
@@ -851,7 +890,9 @@ class ChargingManager:
|
|
|
851
890
|
async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
|
|
852
891
|
"""Atomically reserve and submit one logical charge operation."""
|
|
853
892
|
async with self._operation_lock:
|
|
854
|
-
|
|
893
|
+
result = await self._charge_exclusive(event_name, count)
|
|
894
|
+
self._note_prepaid_charge(event_name, result.charged_count)
|
|
895
|
+
return result
|
|
855
896
|
|
|
856
897
|
async def _charge_exclusive(self, event_name: str, count: int = 1) -> ChargeResult:
|
|
857
898
|
if self._closed:
|
|
@@ -1327,19 +1368,27 @@ async def push_data_and_charge(
|
|
|
1327
1368
|
# default-dataset push that does not pay per row (no event, or only
|
|
1328
1369
|
# free events) stores nothing. Paid pushes go on below, where the
|
|
1329
1370
|
# budget cap stores only the rows that are actually charged.
|
|
1371
|
+
# Rows the Actor already paid for with an earlier charge() (charge first,
|
|
1372
|
+
# then a plain push) are still stored: see ChargingManager._take_unpaid_rows.
|
|
1373
|
+
# The result still reports the limit, so the Actor stops after them.
|
|
1374
|
+
unpaid_past_limit = False
|
|
1330
1375
|
if (
|
|
1331
1376
|
is_ppe
|
|
1332
1377
|
and is_default_dataset
|
|
1333
1378
|
and items
|
|
1334
|
-
and charging_manager.is_charge_limit_reached()
|
|
1335
1379
|
and not _push_is_paid_per_item(charging_manager, event_name, resolved_event)
|
|
1336
1380
|
):
|
|
1337
|
-
charging_manager.
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1381
|
+
unpaid_past_limit = charging_manager.is_charge_limit_reached()
|
|
1382
|
+
allowed = charging_manager._take_unpaid_rows(len(items))
|
|
1383
|
+
if allowed < len(items):
|
|
1384
|
+
charging_manager._note_refused_default_dataset_items(len(items) - allowed)
|
|
1385
|
+
if allowed == 0:
|
|
1386
|
+
return ChargeResult(
|
|
1387
|
+
chargeable_within_limit=charging_manager.compute_chargeable(),
|
|
1388
|
+
charged_count=0,
|
|
1389
|
+
event_charge_limit_reached=True,
|
|
1390
|
+
)
|
|
1391
|
+
items = items[:allowed]
|
|
1343
1392
|
|
|
1344
1393
|
limited_count = len(items)
|
|
1345
1394
|
if is_ppe and resolved_event:
|
|
@@ -1522,7 +1571,8 @@ async def push_data_and_charge(
|
|
|
1522
1571
|
event_charge_limit_reached=any(
|
|
1523
1572
|
result.event_charge_limit_reached for result in results
|
|
1524
1573
|
)
|
|
1525
|
-
or accepted_for_items < len(limited_items)
|
|
1574
|
+
or accepted_for_items < len(limited_items)
|
|
1575
|
+
or unpaid_past_limit,
|
|
1526
1576
|
)
|
|
1527
1577
|
|
|
1528
1578
|
|
|
@@ -1531,10 +1581,170 @@ async def push_data_and_charge(
|
|
|
1531
1581
|
return ChargeResult(
|
|
1532
1582
|
chargeable_within_limit=charging_manager.compute_chargeable(),
|
|
1533
1583
|
charged_count=0,
|
|
1534
|
-
event_charge_limit_reached=
|
|
1584
|
+
event_charge_limit_reached=unpaid_past_limit,
|
|
1535
1585
|
)
|
|
1536
1586
|
|
|
1537
1587
|
|
|
1588
|
+
def _validate_row_charges(charges: Any) -> Dict[str, int]:
|
|
1589
|
+
if not isinstance(charges, dict) or not charges:
|
|
1590
|
+
raise ValueError('charges must be a non-empty dict of {event_name: count_per_row}')
|
|
1591
|
+
out: Dict[str, int] = {}
|
|
1592
|
+
for name, count in charges.items():
|
|
1593
|
+
if not isinstance(name, str) or not name:
|
|
1594
|
+
raise ValueError('charges keys must be event names')
|
|
1595
|
+
if is_default_dataset_item_event(name):
|
|
1596
|
+
raise ValueError(f"Cannot charge for synthetic event '{name}' manually")
|
|
1597
|
+
if not isinstance(count, int) or isinstance(count, bool) or count < 1:
|
|
1598
|
+
raise ValueError(f'charges[{name!r}] must be a positive integer (events per row)')
|
|
1599
|
+
out[name] = count
|
|
1600
|
+
return out
|
|
1601
|
+
|
|
1602
|
+
|
|
1603
|
+
async def push_data_with_charges(
|
|
1604
|
+
charging_manager: ChargingManager,
|
|
1605
|
+
items: Union[Dict[str, Any], List[Dict[str, Any]]],
|
|
1606
|
+
charges: Dict[str, int],
|
|
1607
|
+
is_default_dataset: bool,
|
|
1608
|
+
push_fn: Any,
|
|
1609
|
+
) -> ChargeResult:
|
|
1610
|
+
"""Push rows that each cost several events, atomically per row.
|
|
1611
|
+
|
|
1612
|
+
``charges`` maps event name -> events per row, e.g. ``{'result': 1, 'page': 3}``.
|
|
1613
|
+
A row is stored only when every one of its events was charged: the full set
|
|
1614
|
+
is charged first, rows the budget cannot fully pay for are not stored, and
|
|
1615
|
+
any operation that came back short is voided and re-charged for the rows
|
|
1616
|
+
that are kept, so no row is ever stored partly paid and no event is billed
|
|
1617
|
+
for a row that was not stored. Unpriced events are ignored. On the default
|
|
1618
|
+
dataset the default-dataset-item event (when priced) is added per row.
|
|
1619
|
+
"""
|
|
1620
|
+
charges = _validate_row_charges(charges)
|
|
1621
|
+
rows_in = items if isinstance(items, list) else [items]
|
|
1622
|
+
if not rows_in:
|
|
1623
|
+
return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
|
|
1624
|
+
|
|
1625
|
+
async with charging_manager._operation_lock:
|
|
1626
|
+
if charging_manager._closed:
|
|
1627
|
+
raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
|
|
1628
|
+
voids_confirmed = await charging_manager._retry_pending_voids()
|
|
1629
|
+
|
|
1630
|
+
if charging_manager._pricing_info is None:
|
|
1631
|
+
await _gated_push(push_fn, rows_in)
|
|
1632
|
+
return ChargeResult(chargeable_within_limit={}, charged_count=0, event_charge_limit_reached=False)
|
|
1633
|
+
|
|
1634
|
+
per_row: Dict[str, int] = {
|
|
1635
|
+
name: n for name, n in charges.items()
|
|
1636
|
+
if (charging_manager._calculate_event_price(name) or Decimal('0')) > 0
|
|
1637
|
+
}
|
|
1638
|
+
if is_default_dataset:
|
|
1639
|
+
default_event = charging_manager._config.get('default_dataset_item_event_name') or \
|
|
1640
|
+
resolve_default_dataset_item_event_name(charging_manager._per_event_prices)
|
|
1641
|
+
if default_event and default_event not in per_row and \
|
|
1642
|
+
(charging_manager._calculate_event_price(default_event) or Decimal('0')) > 0:
|
|
1643
|
+
per_row[default_event] = 1
|
|
1644
|
+
if not per_row:
|
|
1645
|
+
# Every listed event is free: a plain (unpaid) push.
|
|
1646
|
+
return await _unpaid_push_locked(charging_manager, rows_in, is_default_dataset, push_fn)
|
|
1647
|
+
if not voids_confirmed and any(not is_default_dataset_item_event(n) for n in per_row):
|
|
1648
|
+
raise RuntimeError('An earlier charge for a failed dataset push could not be voided yet; retry the push')
|
|
1649
|
+
|
|
1650
|
+
# Rows the budget can pay for in full (no one-event overdraw: a row is all or nothing).
|
|
1651
|
+
row_cost = sum(
|
|
1652
|
+
(charging_manager._calculate_event_price(n) or Decimal('0')) * Decimal(c) for n, c in per_row.items()
|
|
1653
|
+
)
|
|
1654
|
+
rows = len(rows_in)
|
|
1655
|
+
if charging_manager._max_total_charge_usd is not None and row_cost > 0:
|
|
1656
|
+
remaining = charging_manager._max_total_charge_usd - charging_manager._total_charged_usd
|
|
1657
|
+
rows = min(rows, max(0, int(remaining / row_cost)))
|
|
1658
|
+
|
|
1659
|
+
ops: List[tuple] = [] # (event, charged, op_id, uncertain)
|
|
1660
|
+
|
|
1661
|
+
async def undo(entries: List[tuple]) -> None:
|
|
1662
|
+
for name, charged, op_id, uncertain in entries:
|
|
1663
|
+
if charged <= 0:
|
|
1664
|
+
continue
|
|
1665
|
+
if op_id is None:
|
|
1666
|
+
charging_manager._unwind_reservation(name, charged)
|
|
1667
|
+
elif uncertain:
|
|
1668
|
+
await charging_manager._cancel_uncertain_charge(name, op_id, charged)
|
|
1669
|
+
else:
|
|
1670
|
+
await charging_manager._void_charge(name, op_id, charged)
|
|
1671
|
+
|
|
1672
|
+
for _attempt in range(4):
|
|
1673
|
+
if rows <= 0:
|
|
1674
|
+
break
|
|
1675
|
+
ops = []
|
|
1676
|
+
short_to: Optional[int] = None
|
|
1677
|
+
for name, n in sorted(per_row.items(), key=lambda kv: (is_default_dataset_item_event(kv[0]), kv[0])):
|
|
1678
|
+
want = n * rows
|
|
1679
|
+
charging_manager._take_last_accepted_operation_id()
|
|
1680
|
+
try:
|
|
1681
|
+
r = await charging_manager._charge_exclusive(name, want)
|
|
1682
|
+
op_id = charging_manager._take_last_accepted_operation_id()
|
|
1683
|
+
ops.append((name, r.charged_count, op_id, False))
|
|
1684
|
+
got = r.charged_count
|
|
1685
|
+
except ChargeUncertainError as unconfirmed:
|
|
1686
|
+
# Not confirmed: the intent stays pending and is re-posted at exit,
|
|
1687
|
+
# so it counts as accepted (items stored, billed once).
|
|
1688
|
+
ops.append((name, unconfirmed.count, unconfirmed.operation_id, True))
|
|
1689
|
+
got = unconfirmed.count
|
|
1690
|
+
if got < want:
|
|
1691
|
+
short_to = got // n
|
|
1692
|
+
break
|
|
1693
|
+
if short_to is None:
|
|
1694
|
+
break
|
|
1695
|
+
# Some event came back short: void this attempt in full and retry with
|
|
1696
|
+
# the rows that event can pay for (a void restores the budget).
|
|
1697
|
+
await undo(ops)
|
|
1698
|
+
ops = []
|
|
1699
|
+
rows = short_to # got < n * rows, so short_to < rows
|
|
1700
|
+
|
|
1701
|
+
if rows <= 0 or not ops:
|
|
1702
|
+
await undo(ops)
|
|
1703
|
+
charging_manager._mark_charge_limit_hit()
|
|
1704
|
+
if is_default_dataset:
|
|
1705
|
+
charging_manager._note_refused_default_dataset_items(len(rows_in))
|
|
1706
|
+
return ChargeResult(
|
|
1707
|
+
chargeable_within_limit=charging_manager.compute_chargeable(),
|
|
1708
|
+
charged_count=0,
|
|
1709
|
+
event_charge_limit_reached=True,
|
|
1710
|
+
)
|
|
1711
|
+
|
|
1712
|
+
kept = rows_in[:rows]
|
|
1713
|
+
if rows < len(rows_in):
|
|
1714
|
+
charging_manager._mark_charge_limit_hit()
|
|
1715
|
+
if is_default_dataset:
|
|
1716
|
+
charging_manager._note_refused_default_dataset_items(len(rows_in) - rows)
|
|
1717
|
+
try:
|
|
1718
|
+
await _gated_push(push_fn, kept)
|
|
1719
|
+
except Exception as push_err:
|
|
1720
|
+
logger.error('Dataset push failed after charges accepted (rows=%s): %s; voiding the charges', rows, push_err)
|
|
1721
|
+
if _push_certainly_failed(push_err):
|
|
1722
|
+
await undo(ops)
|
|
1723
|
+
raise
|
|
1724
|
+
return ChargeResult(
|
|
1725
|
+
chargeable_within_limit=charging_manager.compute_chargeable(),
|
|
1726
|
+
charged_count=rows,
|
|
1727
|
+
event_charge_limit_reached=rows < len(rows_in) or charging_manager.is_charge_limit_reached(),
|
|
1728
|
+
)
|
|
1729
|
+
|
|
1730
|
+
|
|
1731
|
+
async def _unpaid_push_locked(
|
|
1732
|
+
charging_manager: ChargingManager, rows: List[Dict[str, Any]], is_default_dataset: bool, push_fn: Any
|
|
1733
|
+
) -> ChargeResult:
|
|
1734
|
+
"""A push whose rows carry no priced event (caller holds the operation lock)."""
|
|
1735
|
+
past_limit = charging_manager.is_charge_limit_reached()
|
|
1736
|
+
allowed = charging_manager._take_unpaid_rows(len(rows)) if is_default_dataset else len(rows)
|
|
1737
|
+
if allowed < len(rows):
|
|
1738
|
+
charging_manager._note_refused_default_dataset_items(len(rows) - allowed)
|
|
1739
|
+
if allowed > 0:
|
|
1740
|
+
await _gated_push(push_fn, rows[:allowed])
|
|
1741
|
+
return ChargeResult(
|
|
1742
|
+
chargeable_within_limit=charging_manager.compute_chargeable(),
|
|
1743
|
+
charged_count=0,
|
|
1744
|
+
event_charge_limit_reached=past_limit,
|
|
1745
|
+
)
|
|
1746
|
+
|
|
1747
|
+
|
|
1538
1748
|
def _push_is_paid_per_item(
|
|
1539
1749
|
charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
|
|
1540
1750
|
) -> bool:
|
|
@@ -289,3 +289,223 @@ async def test_apify_client_hooks_intercept_default_dataset_past_limit() -> None
|
|
|
289
289
|
assert hooks.should_charge_dataset_pushes(DEFAULT_DATASET_ID) is False
|
|
290
290
|
await mgr.charge('result', 1)
|
|
291
291
|
assert hooks.should_charge_dataset_pushes(DEFAULT_DATASET_ID) is True
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
# ---- charge first, then a plain push (the row was paid before it was pushed) ----
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
@pytest.mark.asyncio
|
|
298
|
+
async def test_charge_then_plain_push_keeps_the_row_whose_charge_hit_the_cap() -> None:
|
|
299
|
+
"""charge('result'); push_data(row): the last paid row must be saved, nothing after it."""
|
|
300
|
+
api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
|
|
301
|
+
mgr = _manager(api)
|
|
302
|
+
store = Store()
|
|
303
|
+
|
|
304
|
+
for n in range(10):
|
|
305
|
+
r = await mgr.charge('result', 1)
|
|
306
|
+
if r.charged_count:
|
|
307
|
+
await _plain_push(mgr, store, [{'n': n}])
|
|
308
|
+
else:
|
|
309
|
+
await _plain_push(mgr, store, [{'n': n, 'unpaid': True}])
|
|
310
|
+
|
|
311
|
+
assert [i['n'] for i in store.items] == [0, 1, 2]
|
|
312
|
+
assert api.spent == Decimal('0.03')
|
|
313
|
+
assert mgr.is_charge_limit_reached() is True
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
@pytest.mark.asyncio
|
|
317
|
+
async def test_prepaid_rows_cap_a_bigger_plain_push() -> None:
|
|
318
|
+
api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.02'))
|
|
319
|
+
mgr = _manager(api, max_usd='0.02')
|
|
320
|
+
store = Store()
|
|
321
|
+
|
|
322
|
+
await mgr.charge('result', 2)
|
|
323
|
+
assert mgr.is_charge_limit_reached() is True
|
|
324
|
+
r = await _plain_push(mgr, store, [{'n': 1}, {'n': 2}, {'n': 3}])
|
|
325
|
+
assert [i['n'] for i in store.items] == [1, 2]
|
|
326
|
+
assert r.event_charge_limit_reached is True
|
|
327
|
+
assert mgr._refused_default_dataset_items == 1
|
|
328
|
+
# Balance used up: the next plain row is refused.
|
|
329
|
+
await _plain_push(mgr, store, [{'n': 4}])
|
|
330
|
+
assert len(store.items) == 2
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
@pytest.mark.asyncio
|
|
334
|
+
async def test_push_then_charge_actor_gets_no_free_row_after_the_cap() -> None:
|
|
335
|
+
"""Balance never goes positive for push-then-charge, so the cap still stops it."""
|
|
336
|
+
api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
|
|
337
|
+
mgr = _manager(api)
|
|
338
|
+
store = Store()
|
|
339
|
+
|
|
340
|
+
for n in range(6):
|
|
341
|
+
await _plain_push(mgr, store, [{'n': n}])
|
|
342
|
+
await mgr.charge('result', 1)
|
|
343
|
+
|
|
344
|
+
assert [i['n'] for i in store.items] == [0, 1, 2]
|
|
345
|
+
assert mgr._prepaid_row_balance <= 0
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
@pytest.mark.asyncio
|
|
349
|
+
async def test_unpriced_or_free_charges_do_not_prepay_rows() -> None:
|
|
350
|
+
api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
|
|
351
|
+
mgr = _manager(api, prices={'result': Decimal('0.01'), 'free-event': Decimal('0')}, max_usd='0.01')
|
|
352
|
+
store = Store()
|
|
353
|
+
|
|
354
|
+
await mgr.charge('result', 1)
|
|
355
|
+
await _plain_push(mgr, store, [{'n': 1}]) # paid by the charge above
|
|
356
|
+
await mgr.charge('free-event', 3)
|
|
357
|
+
await mgr.charge('not-in-pricing', 3)
|
|
358
|
+
await _plain_push(mgr, store, [{'n': 2}])
|
|
359
|
+
assert [i['n'] for i in store.items] == [1]
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
@pytest.mark.asyncio
|
|
363
|
+
async def test_storage_client_push_after_prepaying_charge_is_stored() -> None:
|
|
364
|
+
api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
|
|
365
|
+
mgr = _manager(api, max_usd='0.01')
|
|
366
|
+
set_active_charging_manager(mgr)
|
|
367
|
+
client, pushed = _dataset_client(DEFAULT_DATASET_ID)
|
|
368
|
+
|
|
369
|
+
await mgr.charge('result', 1) # uses the whole budget, pays for one row
|
|
370
|
+
await client.push_data([{'n': 1}, {'n': 2}]) # two rows, one paid: whole batch refused
|
|
371
|
+
await client.push_data([{'n': 3}]) # one paid row: stored
|
|
372
|
+
await client.push_data([{'n': 4}]) # nothing left: refused
|
|
373
|
+
|
|
374
|
+
assert pushed == [(DEFAULT_DATASET_ID, [{'n': 3}])]
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
@pytest.mark.asyncio
|
|
378
|
+
async def test_partial_acceptance_saves_only_the_accepted_rows() -> None:
|
|
379
|
+
api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
|
|
380
|
+
mgr = _manager(api, max_usd='5')
|
|
381
|
+
store = Store()
|
|
382
|
+
|
|
383
|
+
c = await mgr.charge('result', 5)
|
|
384
|
+
assert c.charged_count == 3
|
|
385
|
+
r = await _plain_push(mgr, store, [{'n': n} for n in range(1, 6)])
|
|
386
|
+
assert [i['n'] for i in store.items] == [1, 2, 3]
|
|
387
|
+
assert r.event_charge_limit_reached is True
|
|
388
|
+
await _plain_push(mgr, store, [{'n': 6}])
|
|
389
|
+
assert len(store.items) == 3
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
@pytest.mark.asyncio
|
|
393
|
+
async def test_refused_charge_402_then_plain_push_is_not_saved() -> None:
|
|
394
|
+
api = FakeChargeApi(price=Decimal('0.01'), max_usd=None, reject_after=2)
|
|
395
|
+
mgr = _manager(api, max_usd='5')
|
|
396
|
+
store = Store()
|
|
397
|
+
|
|
398
|
+
for n in range(4):
|
|
399
|
+
await mgr.charge('result', 1)
|
|
400
|
+
await _plain_push(mgr, store, [{'n': n}])
|
|
401
|
+
|
|
402
|
+
assert [i['n'] for i in store.items] == [0, 1]
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
# ---- atomic multi-event push: push_data(row, charges={...}) ----
|
|
406
|
+
|
|
407
|
+
from scrapely.charging import push_data_with_charges # noqa: E402
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
class MultiApi(FakeChargeApi):
|
|
411
|
+
"""Charge endpoint priced per event, with void support (by Idempotency-Key)."""
|
|
412
|
+
|
|
413
|
+
def __init__(self, prices: Dict[str, Decimal], max_usd: Optional[Decimal]) -> None:
|
|
414
|
+
super().__init__(price=Decimal('0'), max_usd=max_usd)
|
|
415
|
+
self.prices = prices
|
|
416
|
+
self.ops: Dict[str, tuple] = {}
|
|
417
|
+
self.voids: List[str] = []
|
|
418
|
+
|
|
419
|
+
async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **_: Any) -> Dict[str, Any]:
|
|
420
|
+
key = (headers or {}).get('Idempotency-Key', '')
|
|
421
|
+
if path.endswith('/charge/void'):
|
|
422
|
+
name, accepted = self.ops.pop(key)
|
|
423
|
+
self.spent -= self.prices[name] * accepted
|
|
424
|
+
self.voids.append(name)
|
|
425
|
+
return {}
|
|
426
|
+
name = (json_data or {}).get('eventName')
|
|
427
|
+
count = int((json_data or {}).get('count', 1))
|
|
428
|
+
price = self.prices[name]
|
|
429
|
+
accepted = count
|
|
430
|
+
if self.max_usd is not None and price > 0:
|
|
431
|
+
affordable = int((self.max_usd - self.spent) / price)
|
|
432
|
+
if affordable < 1:
|
|
433
|
+
raise HttpError(402, 'charge-limit-exceeded')
|
|
434
|
+
accepted = min(count, affordable)
|
|
435
|
+
self.spent += price * accepted
|
|
436
|
+
self.ops[key] = (name, accepted)
|
|
437
|
+
return {'acceptedCount': accepted, 'requestedCount': count}
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
PRICES = {'result': Decimal('0.01'), 'page': Decimal('0.002')} # one row = 1 result + 3 pages = $0.016
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _multi(max_usd: str, server_max: Optional[str] = None):
|
|
444
|
+
api = MultiApi(PRICES, Decimal(server_max or max_usd))
|
|
445
|
+
return api, _manager(api, prices=dict(PRICES), max_usd=max_usd)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
@pytest.mark.asyncio
|
|
449
|
+
async def test_multi_event_exact_budget_stores_every_paid_row() -> None:
|
|
450
|
+
api, mgr = _multi('0.048') # exactly 3 rows
|
|
451
|
+
store = Store()
|
|
452
|
+
r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
|
|
453
|
+
assert r.charged_count == 3
|
|
454
|
+
assert [i['n'] for i in store.items] == [0, 1, 2]
|
|
455
|
+
assert api.spent == Decimal('0.048')
|
|
456
|
+
assert mgr.is_charge_limit_reached() is True
|
|
457
|
+
again = await push_data_with_charges(mgr, [{'n': 9}], {'result': 1, 'page': 3}, True, store.push)
|
|
458
|
+
assert again.charged_count == 0 and again.event_charge_limit_reached is True
|
|
459
|
+
assert len(store.items) == 3 and api.spent == Decimal('0.048')
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
@pytest.mark.asyncio
|
|
463
|
+
async def test_multi_event_partial_budget_stores_only_fully_paid_rows() -> None:
|
|
464
|
+
api, mgr = _multi('0.040') # 2.5 rows: only 2 fully paid
|
|
465
|
+
store = Store()
|
|
466
|
+
r = await push_data_with_charges(mgr, [{'n': n} for n in range(5)], {'result': 1, 'page': 3}, True, store.push)
|
|
467
|
+
assert r.charged_count == 2 and r.event_charge_limit_reached is True
|
|
468
|
+
assert [i['n'] for i in store.items] == [0, 1]
|
|
469
|
+
assert api.spent == Decimal('0.032') # never a part-paid row
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
@pytest.mark.asyncio
|
|
473
|
+
async def test_multi_event_server_cap_short_voids_and_recharges() -> None:
|
|
474
|
+
"""Local budget says 3 rows; the platform (usage ate the budget) caps 'result' at 2."""
|
|
475
|
+
api, mgr = _multi('1', server_max='0.032')
|
|
476
|
+
store = Store()
|
|
477
|
+
r = await push_data_with_charges(mgr, [{'n': n} for n in range(3)], {'result': 1, 'page': 3}, True, store.push)
|
|
478
|
+
# The platform's remaining budget is only learnt from the short answer, so the
|
|
479
|
+
# retry keeps what that answer proves affordable: fewer rows, never a part-paid one.
|
|
480
|
+
assert 1 <= r.charged_count <= 2
|
|
481
|
+
assert len(store.items) == r.charged_count
|
|
482
|
+
assert api.spent == Decimal('0.016') * r.charged_count
|
|
483
|
+
assert api.voids # the short attempt was voided, not left part-paid
|
|
484
|
+
assert sorted(n for n, _ in api.ops.values()) == ['page', 'result']
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
@pytest.mark.asyncio
|
|
488
|
+
async def test_multi_event_refusal_stores_nothing_and_bills_nothing() -> None:
|
|
489
|
+
api, mgr = _multi('0.015') # less than one full row
|
|
490
|
+
store = Store()
|
|
491
|
+
r = await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push)
|
|
492
|
+
assert r.charged_count == 0 and r.event_charge_limit_reached is True
|
|
493
|
+
assert store.items == [] and api.spent == Decimal('0')
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
@pytest.mark.asyncio
|
|
497
|
+
async def test_multi_event_push_does_not_touch_prepaid_balance() -> None:
|
|
498
|
+
api, mgr = _multi('0.026')
|
|
499
|
+
store = Store()
|
|
500
|
+
await mgr.charge('result', 1) # prepays one plain row
|
|
501
|
+
await push_data_with_charges(mgr, [{'n': 1}], {'result': 1, 'page': 3}, True, store.push) # $0.016, paid itself
|
|
502
|
+
assert mgr._prepaid_row_balance == 1
|
|
503
|
+
await _plain_push(mgr, store, [{'n': 2}]) # the prepaid row
|
|
504
|
+
assert [i['n'] for i in store.items] == [1, 2]
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def test_multi_event_validation() -> None:
|
|
508
|
+
from scrapely.charging import _validate_row_charges
|
|
509
|
+
for bad in ({}, {'result': 0}, {'result': 1.5}, {'apify-default-dataset-item': 1}, {'': 1}, None):
|
|
510
|
+
with pytest.raises(ValueError):
|
|
511
|
+
_validate_row_charges(bad)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/_file_system/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_dataset_client.py
RENAMED
|
File without changes
|
{scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_kvs_client.py
RENAMED
|
File without changes
|
{scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_rq_client.py
RENAMED
|
File without changes
|
{scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely/storage/scrapely_storage_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
{scrapely_python-0.1.16 → scrapely_python-0.1.18}/src/scrapely_python.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|