scrapely-python 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.3'
+__version__ = '0.1.4'
 __all__ = [
     # Core
     'Actor',

scrapely/charging.py

@@ -199,7 +199,7 @@ class ChargingManager:
         # (push_data_and_charge voids it when the push then fails).
         self._last_accepted_operation_id: Optional[str] = None
         # Voids not yet confirmed by the platform; retried before the next push and at exit.
-        self._pending_voids: List[tuple[str, str, int]] = []
+        self._pending_voids: List[tuple[str, str, int, bool]] = []
 
     def close(self) -> None:
         """Mark the manager closed; subsequent charge/push operations are rejected."""
@@ -439,7 +439,18 @@ class ChargingManager:
         self._last_accepted_operation_id = None
         return op_id
 
-    async def _void_charge(self, event_name: str, operation_id: str, reserved: int) -> bool:
+    async def _cancel_uncertain_charge(self, event_name: str, operation_id: str, reserved: int) -> bool:
+        """Cancel an unconfirmed charge for items that were then not stored.
+
+        Drops the pending intent (so the exit flush never posts it) and voids it in
+        case the original request did apply; a 404 means it never applied.
+        """
+        self._drop_pending(operation_id)
+        return await self._void_charge(event_name, operation_id, reserved, not_found_means_voided=True)
+
+    async def _void_charge(
+        self, event_name: str, operation_id: str, reserved: int, not_found_means_voided: bool = False
+    ) -> bool:
         """Reverse an accepted charge whose items were never stored, then unwind locally.
 
         POST /v2/actor-runs/{run}/charge/void with the charge's Idempotency-Key.
@@ -458,13 +469,19 @@ class ChargingManager:
                 return True
             except Exception as err:  # noqa: BLE001
                 status = _http_status_of(err)
+                # Only the void route's own answer proves it never applied; a server
+                # without the route (older deployment) also answers 404.
+                if status == 404 and not_found_means_voided and getattr(err, 'type', None) == 'charge-not-found':
+                    # The unconfirmed charge never reached the ledger: nothing billed.
+                    self._unwind_reservation(event_name, reserved)
+                    return True
                 # 404: nothing to void; 409: already settled; 4xx auth: permanent.
                 if status is not None and 400 <= status < 500:
                     return False
                 if attempt + 1 < self._POST_ATTEMPTS:
                     await asyncio.sleep(0.25 * (attempt + 1))
         # Outcome unknown: keep the reservation and retry before the next push / at exit.
-        self._pending_voids.append((event_name, operation_id, reserved))
+        self._pending_voids.append((event_name, operation_id, reserved, not_found_means_voided))
         logger.error('Void of charge %s (%r) not confirmed yet; will retry', operation_id, event_name)
         return False
 
@@ -474,8 +491,8 @@ class ChargingManager:
         Returns False while any earlier void is still unconfirmed.
         """
         pending, self._pending_voids = self._pending_voids, []
-        for event_name, operation_id, reserved in pending:
-            await self._void_charge(event_name, operation_id, reserved)
+        for event_name, operation_id, reserved, not_found_means_voided in pending:
+            await self._void_charge(event_name, operation_id, reserved, not_found_means_voided)
         return not self._pending_voids
 
     def _quarantine_stale_pending(self) -> None:
@@ -1158,19 +1175,39 @@ async def push_data_and_charge(
             # Synthetic storage event first (local-only); then explicit billable event.
             reserved_in_op: list[tuple[str, int]] = []
             operation_ids: list[Optional[str]] = []
+            uncertain_ops: set[str] = set()
+
+            async def charge_one(name: str) -> None:
+                charging_manager._take_last_accepted_operation_id()
+                try:
+                    r = await charging_manager._charge_exclusive(name, len(limited_items))
+                    op_id = charging_manager._take_last_accepted_operation_id()
+                except ChargeUncertainError as unconfirmed:
+                    # The platform did not confirm the charge (5xx / timeouts after
+                    # retries). Never drop the items for that: the intent stays
+                    # pending with its idempotency key and is re-posted by the exit
+                    # flush, so the items are stored now and billed once.
+                    logger.warning(
+                        'Charge for %r not confirmed yet (%s); storing the items and retrying the charge later',
+                        name,
+                        unconfirmed.operation_id,
+                    )
+                    r = ChargeResult(
+                        chargeable_within_limit=charging_manager.compute_chargeable(),
+                        charged_count=unconfirmed.count,
+                        event_charge_limit_reached=False,
+                    )
+                    op_id = unconfirmed.operation_id
+                    uncertain_ops.add(op_id)
+                results.append(r)
+                reserved_in_op.append((name, r.charged_count))
+                operation_ids.append(op_id)
+
             try:
                 if default_event:
-                    charging_manager._take_last_accepted_operation_id()
-                    r = await charging_manager._charge_exclusive(default_event, len(limited_items))
-                    results.append(r)
-                    reserved_in_op.append((default_event, r.charged_count))
-                    operation_ids.append(charging_manager._take_last_accepted_operation_id())
+                    await charge_one(default_event)
                 if event_name and event_name != default_event:
-                    charging_manager._take_last_accepted_operation_id()
-                    r = await charging_manager._charge_exclusive(event_name, len(limited_items))
-                    results.append(r)
-                    reserved_in_op.append((event_name, r.charged_count))
-                    operation_ids.append(charging_manager._take_last_accepted_operation_id())
+                    await charge_one(event_name)
             except Exception:
                 for name, reserved in reserved_in_op:
                     if reserved > 0:
@@ -1192,11 +1229,31 @@ async def push_data_and_charge(
                 )
                 # Unwind any event that reserved more than we will persist
                 # (synthetic default-dataset is local-only and otherwise sticks).
+                platform_op_names = {
+                    name for (name, _), op_id in zip(reserved_in_op, operation_ids) if op_id
+                }
                 for name, result in charged_events:
                     if result.charged_count > accepted_for_items:
+                        # Nothing stored and a platform charge was made: the void
+                        # below releases its local count once the platform confirms.
+                        if accepted_for_items <= 0 and name in platform_op_names:
+                            continue
                         charging_manager._unwind_reservation(
                             name, result.charged_count - accepted_for_items
                         )
+                if accepted_for_items <= 0:
+                    # Nothing will be stored (another event of this push was capped
+                    # to zero): undo every platform charge this operation made, so
+                    # no result is billed for items that are never saved. Local
+                    # counts were unwound just above.
+                    for (name, reserved), op_id in zip(reserved_in_op, operation_ids):
+                        if reserved <= 0 or not op_id:
+                            continue
+                        # Original count: released only when the platform confirms.
+                        if op_id in uncertain_ops:
+                            await charging_manager._cancel_uncertain_charge(name, op_id, reserved)
+                        else:
+                            await charging_manager._void_charge(name, op_id, reserved)
                 if accepted_for_items > 0:
                     try:
                         await push_fn(limited_items[:accepted_for_items])
@@ -1218,7 +1275,10 @@ async def push_data_and_charge(
                             kept = min(accepted_for_items, next(r.charged_count for n, r in charged_events if n == name))
                             if kept <= 0:
                                 continue
-                            if op_id:
+                            if op_id and op_id in uncertain_ops:
+                                # Never re-post a charge for items that were not stored.
+                                await charging_manager._cancel_uncertain_charge(name, op_id, kept)
+                            elif op_id:
                                 await charging_manager._void_charge(name, op_id, kept)
                             elif is_default_dataset_item_event(name):
                                 charging_manager._unwind_reservation(name, kept)

{scrapely_python-0.1.3.dist-info → scrapely_python-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.3
+Version: 0.1.4
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT

{scrapely_python-0.1.3.dist-info → scrapely_python-0.1.4.dist-info}/RECORD

@@ -1,8 +1,8 @@
-scrapely/__init__.py,sha256=vJx9b5JOO4ehkpvB03Xjoj7dCIzw4lgLggG-0jWrkM0,5745
+scrapely/__init__.py,sha256=nVwtGU9e9kfzoiX6MdfbK4U-p88J7eMDUkqNkpKY1Ug,5745
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
 scrapely/actor.py,sha256=QczHOj2GVQnMbQZ_76vjjJNHxPrinWB_WCdZ6_41l00,75126
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
-scrapely/charging.py,sha256=eoOWnbtUPcfuh96Ss6XZmOveLInprs6EjSAtrGrCd9A,53364
+scrapely/charging.py,sha256=MFz_Z4qJ-N3qLffleQb2BxjomygM3UMvHSBw1HabG80,56857
 scrapely/charging_client.py,sha256=tDtlPwLSVFvoRa2NBgNjyfHkAUM77SSIyYT2ACgpDm8,3710
 scrapely/configuration.py,sha256=YUnbft9JeWlVTgWalr-CgJCiyJ_e_rgoQItVPvjeWRs,15406
 scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
@@ -37,7 +37,7 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
 scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
 scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
 scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
-scrapely_python-0.1.3.dist-info/METADATA,sha256=pp3DbPQObzUEt1NXKP6prb8mCreU6crymE1QG2-EFSM,3104
-scrapely_python-0.1.3.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-scrapely_python-0.1.3.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.3.dist-info/RECORD,,
+scrapely_python-0.1.4.dist-info/METADATA,sha256=ACM2A5RD1zJ8JV3cEY0jZigW3dpvsjQyiwJBFvKlLv4,3104
+scrapely_python-0.1.4.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.4.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.4.dist-info/RECORD,,