scrapely-python 0.1.21__tar.gz → 0.1.22__tar.gz

{scrapely_python-0.1.21/src/scrapely_python.egg-info → scrapely_python-0.1.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.21
+Version: 0.1.22
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.21 → scrapely_python-0.1.22}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.21"
+version = "0.1.22"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.21 → scrapely_python-0.1.22}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.21'
+__version__ = '0.1.22'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.21 → scrapely_python-0.1.22}/src/scrapely/actor.py

@@ -784,6 +784,7 @@ class Actor(metaclass=_ActorMeta):
         event_name: Optional[str] = None,
         charges: Optional[Dict[str, int]] = None,
         charge_per: str = 'row',
+        dataset: Optional[Union[str, Dataset]] = None,
     ) -> ChargeResult:
         """Push data to the default dataset, charging pay-per-event items.
 
@@ -807,6 +808,12 @@ class Actor(metaclass=_ActorMeta):
                 charges them once for the whole push, e.g.
                 ``push_data(page_rows, charges={'document': 1}, charge_per='batch')``:
                 all rows are stored if the charge is accepted, none if it is refused.
+            dataset: Push to this named dataset (name, or a Dataset from ``open_dataset``)
+                instead of the default one, with the same charge-first semantics:
+                ``push_data(rows, charged_event_name='comment', dataset=f'comments-{run_id}')``.
+                Rows of a named dataset carry no implicit default-dataset-item event and are
+                not gated by the default dataset's spending-limit safety net; charged pushes
+                store only the rows that were charged.
 
         Returns:
             ChargeResult describing how many items were charged and pushed.
@@ -827,6 +834,13 @@ class Actor(metaclass=_ActorMeta):
         if cls._default_dataset is None:
             cls._default_dataset = await cls.open_dataset()
 
+        target = cls._default_dataset
+        if dataset is not None:
+            target = dataset if isinstance(dataset, Dataset) else await cls.open_dataset(name=dataset)
+        is_default = target is cls._default_dataset or (
+            getattr(target, 'id', None) is not None and target.id == getattr(cls._default_dataset, 'id', None)
+        )
+
         items = data if isinstance(data, list) else [data]
         charging_manager = cls.get_charging_manager()
         if charges is not None:
@@ -836,8 +850,8 @@ class Actor(metaclass=_ActorMeta):
                 charging_manager,
                 items,
                 charges,
-                True,  # Actor.push_data always targets the default dataset
-                cls._default_dataset.push_data,
+                is_default,
+                target.push_data,
                 charge_per=charge_per,
             )
         if charge_per != 'row':
@@ -846,8 +860,8 @@ class Actor(metaclass=_ActorMeta):
             charging_manager,
             items,
             event_name,
-            True,  # Actor.push_data always targets the default dataset
-            cls._default_dataset.push_data,
+            is_default,
+            target.push_data,
         )
 
     @classmethod

{scrapely_python-0.1.21 → scrapely_python-0.1.22/src/scrapely_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.21
+Version: 0.1.22
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.21 → scrapely_python-0.1.22}/tests/test_charge_limit_guard.py

@@ -728,3 +728,98 @@ def test_charge_per_validation() -> None:
     api, mgr = _doc('1')
     with pytest.raises(ValueError):
         asyncio.run(push_data_with_charges(mgr, _pages(1), {'document': 1}, True, Store().push, charge_per='page'))
+
+
+# ---- charged pushes to a NAMED dataset (add-on rows, e.g. comments-<runId>-<videoId>) ----
+
+
+@pytest.mark.asyncio
+async def test_named_dataset_charged_push_stores_only_charged_rows() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.03'))
+    mgr = _manager(api)
+    named = Store()
+    r = await push_data_and_charge(mgr, [{'c': n} for n in range(5)], 'result', False, named.push)
+    assert r.charged_count == 3 and r.event_charge_limit_reached is True
+    assert [i['c'] for i in named.items] == [0, 1, 2]
+    assert api.spent == Decimal('0.03')
+    assert mgr._prepaid_row_balance == 0, 'paid named rows neither use nor add prepaid rows'
+
+
+@pytest.mark.asyncio
+async def test_named_dataset_failed_push_voids_the_charge() -> None:
+    api = MultiApi({'comment': Decimal('0.01')}, Decimal('1'))
+    mgr = _manager(api, prices={'comment': Decimal('0.01')}, max_usd='1')
+
+    class Boom(HttpError):
+        pass
+
+    async def failing_push(rows: List[Dict[str, Any]]) -> None:
+        raise Boom(400, 'invalid item')  # the storage API answered: nothing stored
+
+    with pytest.raises(Boom):
+        await push_data_with_charges(mgr, [{'c': 1}, {'c': 2}], {'comment': 1}, False, failing_push)
+    assert api.spent == 0 and api.voids, 'paid rows that were not stored are not billed'
+
+
+@pytest.mark.asyncio
+async def test_named_dataset_multi_event_and_batch_modes() -> None:
+    api, mgr = _multi('0.04')
+    named = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(5)], {'result': 1, 'page': 3}, False, named.push)
+    assert r.charged_count == 2 and len(named.items) == 2
+    api2, mgr2 = _doc('1')
+    named2 = Store()
+    r2 = await push_data_with_charges(mgr2, _pages(4), {'document': 1}, False, named2.push, charge_per='batch')
+    assert r2.charged_count == 4 and len(named2.items) == 4 and api2.spent == Decimal('0.01')
+
+
+@pytest.mark.asyncio
+async def test_named_dataset_plain_push_is_not_gated_by_the_default_safety_net() -> None:
+    api = FakeChargeApi(price=Decimal('0.01'), max_usd=Decimal('0.01'))
+    mgr = _manager(api, max_usd='0.01')
+    await mgr.charge('result', 1)
+    assert mgr.is_charge_limit_reached() is True
+    named = Store()
+    await push_data_and_charge(mgr, [{'meta': 1}], None, False, named.push)
+    assert len(named.items) == 1
+    assert mgr._prepaid_row_balance == 1, 'a named plain push does not use the default dataset prepaid rows'
+
+
+@pytest.mark.asyncio
+async def test_actor_push_data_routes_dataset_argument(monkeypatch: pytest.MonkeyPatch) -> None:
+    from scrapely import actor as actor_mod
+    calls: List[Any] = []
+
+    class FakeDs:
+        def __init__(self, ds_id: str) -> None:
+            self.id = ds_id
+            self.rows: List[Any] = []
+
+        async def push_data(self, rows: Any) -> None:
+            self.rows.extend(rows)
+
+    default, named = FakeDs('default-id'), FakeDs('named-id')
+    A = actor_mod.Actor
+    monkeypatch.setattr(A, '_ensure_initialized', classmethod(lambda cls, name: None))
+    monkeypatch.setattr(A, '_default_dataset', default)
+
+    async def fake_open(cls, name_or_id=None, options=None, *, id=None, name=None, alias=None, force_cloud=False):
+        calls.append(name)
+        return named
+
+    monkeypatch.setattr(A, 'open_dataset', classmethod(fake_open))
+    monkeypatch.setattr(actor_mod, 'Dataset', FakeDs)
+    seen: List[Any] = []
+
+    async def fake_pdc(mgr, items, event, is_default, push_fn):
+        seen.append((event, is_default))
+        await push_fn(items)
+        return 'ok'
+
+    monkeypatch.setattr(actor_mod, 'push_data_and_charge', fake_pdc)
+    monkeypatch.setattr(A, 'get_charging_manager', classmethod(lambda cls: None))
+    await A.push_data([{'a': 1}], charged_event_name='comment', dataset='comments-run-1')
+    await A.push_data([{'b': 1}], charged_event_name='result')
+    assert calls == ['comments-run-1']
+    assert seen == [('comment', False), ('result', True)]
+    assert named.rows == [{'a': 1}] and default.rows == [{'b': 1}]