scrapely-python 0.1.26__tar.gz → 0.1.28__tar.gz

{scrapely_python-0.1.26/src/scrapely_python.egg-info → scrapely_python-0.1.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.26
+Version: 0.1.28
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.26 → scrapely_python-0.1.28}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.26"
+version = "0.1.28"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.26 → scrapely_python-0.1.28}/src/scrapely/__init__.py

@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any
 # the Crawlee patches are applied when Crawlee's browser modules load.
 from .patch_browser_pool import set_cdp_url  # noqa: F401
 
-# Crawlee is NOT imported here: it costs ~1 s per run under gVisor and many
+# Crawlee is NOT imported here: it costs ~1 s per run on the platform and many
 # actors never touch it. Everything that needs Crawlee (the re-exported Crawlee
 # classes, storage clients, events, request loaders) is resolved on first
 # attribute access by ``__getattr__`` below; ``Actor`` imports what it needs on
@@ -227,7 +227,7 @@ if TYPE_CHECKING:  # Static analysers and IDEs see the lazy names as plain impor
         set_api_storage,
     )
 
-__version__ = '0.1.26'
+__version__ = '0.1.28'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.26 → scrapely_python-0.1.28}/src/scrapely/actor.py

@@ -68,7 +68,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 # Names that need Crawlee, resolved on first use instead of at import: importing
-# Crawlee costs ~1 s per run under gVisor. They stay module attributes
+# Crawlee costs ~1 s per run on the platform. They stay module attributes
 # (``scrapely.actor.Dataset`` works and can be monkeypatched): ``_lazy`` returns
 # the module-global value when one is set, else imports and caches it.
 _LAZY_ATTRS: Dict[str, Tuple[str, str]] = {

{scrapely_python-0.1.26 → scrapely_python-0.1.28}/src/scrapely/patch_browser_pool.py

@@ -27,10 +27,10 @@ Mirrors the Node SDK ``patch-browser-pool.ts``:
 - Unpatched builds (``SCRAPELY_BROWSER_PATCH=0``, set by the worker when the
   actor's browser patch was off at build time) keep their browser binaries:
   every patched entry point falls through to the original local launch, with
-  gVisor-safe Chromium args under ``SCRAPELY_CONTAINER_RUNTIME=runsc``.
+  sandbox-safe Chromium args when running on the platform.
 - Soft no-op when Playwright/Crawlee browsers are not installed (e.g. HTTP-only
   actors).
-- Crawlee is never imported by this module (it costs ~1 s under gVisor). The
+- Crawlee is never imported by this module (it costs ~1 s on the platform). The
   Crawlee patches are applied at once when Crawlee's browser / Playwright
   crawler modules are already loaded, otherwise by a post-import hook the
   moment those modules finish loading — before actor code can use the classes.
@@ -121,8 +121,8 @@ def _browser_patch_disabled() -> bool:
     return os.environ.get('SCRAPELY_BROWSER_PATCH') == '0'
 
 
-# gVisor lacks crashpad support; ReadonlyRootfs + tmpfs /tmp loses crash dirs.
-_GVISOR_CHROME_ARGS = (
+# Platform sandbox: no crash reporter; read-only rootfs + tmpfs /tmp loses crash dirs.
+_SANDBOX_CHROME_ARGS = (
     '--disable-breakpad',
     '--disable-crash-reporter',
     '--disable-dev-shm-usage',
@@ -130,13 +130,13 @@ _GVISOR_CHROME_ARGS = (
 )
 
 
-def _with_gvisor_args(options: Optional[dict]) -> dict:
-    """Copy of launch options with gVisor-safe Chromium args when under runsc."""
+def _with_sandbox_args(options: Optional[dict]) -> dict:
+    """Copy of launch options with sandbox-safe Chromium args when on the platform."""
     out = dict(options or {})
-    if os.environ.get('SCRAPELY_CONTAINER_RUNTIME') != 'runsc':
+    if not _is_at_home():
         return out
     args = list(out.get('args') or [])
-    for arg in _GVISOR_CHROME_ARGS:
+    for arg in _SANDBOX_CHROME_ARGS:
         if arg not in args:
             args.append(arg)
     out['args'] = args
@@ -678,7 +678,7 @@ def _patch_raw_browser_type(browser_type: Any, chromium: Any, name: str, is_asyn
         async def launch(**kwargs: Any) -> Any:
             browser_configuration = _take_browser_configuration(kwargs)
             if _browser_patch_disabled():
-                return await orig_launch(**_with_gvisor_args(kwargs))
+                return await orig_launch(**_with_sandbox_args(kwargs))
             cdp_url = _raw_launch_cdp_url(kwargs, browser_configuration)
             if not cdp_url:
                 if not _is_at_home():
@@ -691,7 +691,7 @@ def _patch_raw_browser_type(browser_type: Any, chromium: Any, name: str, is_asyn
         async def launch_persistent_context(*args: Any, **kwargs: Any) -> Any:
             browser_configuration = _take_browser_configuration(kwargs)
             if _browser_patch_disabled():
-                return await orig_lpc(*args, **_with_gvisor_args(kwargs))
+                return await orig_lpc(*args, **_with_sandbox_args(kwargs))
             cdp_url = _raw_launch_cdp_url(kwargs, browser_configuration)
             if not cdp_url:
                 if not _is_at_home():
@@ -710,7 +710,7 @@ def _patch_raw_browser_type(browser_type: Any, chromium: Any, name: str, is_asyn
         def launch(**kwargs: Any) -> Any:
             browser_configuration = _take_browser_configuration(kwargs)
             if _browser_patch_disabled():
-                return orig_launch(**_with_gvisor_args(kwargs))
+                return orig_launch(**_with_sandbox_args(kwargs))
             cdp_url = _raw_launch_cdp_url(kwargs, browser_configuration)
             if not cdp_url:
                 if not _is_at_home():
@@ -722,7 +722,7 @@ def _patch_raw_browser_type(browser_type: Any, chromium: Any, name: str, is_asyn
         def launch_persistent_context(*args: Any, **kwargs: Any) -> Any:
             browser_configuration = _take_browser_configuration(kwargs)
             if _browser_patch_disabled():
-                return orig_lpc(*args, **_with_gvisor_args(kwargs))
+                return orig_lpc(*args, **_with_sandbox_args(kwargs))
             cdp_url = _raw_launch_cdp_url(kwargs, browser_configuration)
             if not cdp_url:
                 if not _is_at_home():
@@ -847,7 +847,7 @@ def _apply_pyppeteer_patch() -> None:
             options = dict(options)
             browser_configuration = _take_browser_configuration(options) or browser_configuration
         if _browser_patch_disabled():
-            return await _orig_launch(_with_gvisor_args({**(options or {}), **kwargs}))
+            return await _orig_launch(_with_sandbox_args({**(options or {}), **kwargs}))
         cdp_url = get_cdp_url()
         if not cdp_url:
             if not _is_at_home():
@@ -876,7 +876,7 @@ def _apply_pyppeteer_patch() -> None:
 
 # ─── Crawlee patches without importing Crawlee ───────────────────────────────
 #
-# Importing Crawlee costs ~1 s per run under gVisor, so this module never
+# Importing Crawlee costs ~1 s per run on the platform, so this module never
 # imports it. The Crawlee patches mutate classes in place, so they only have to
 # run before actor code uses those classes: they are applied immediately when
 # the defining modules are already loaded, otherwise by a post-import hook the

{scrapely_python-0.1.26 → scrapely_python-0.1.28}/src/scrapely/solver.py

@@ -88,7 +88,7 @@ DEGRADED_STATUS_MESSAGE = "Site protection temporarily unavailable, results may
 # Codes the platform answers with when a vendor cannot be solved right now.
 UNAVAILABLE_CODES = ("vendor_degraded", "vendor_down", "provider_unavailable", "vendor_unsupported", "solver_not_enabled")
 
-_COMPAT_SUFFIXES = ("/compat/v1", "/compat/xhrdev")
+_COMPAT_SUFFIXES = ("/compat/v1",)
 _STATUS_TTL_SECS = 15.0
 _MAX_BODY = 4 * 1024 * 1024
 

{scrapely_python-0.1.26 → scrapely_python-0.1.28/src/scrapely_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.26
+Version: 0.1.28
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.26 → scrapely_python-0.1.28}/tests/test_lazy_crawlee.py

@@ -1,6 +1,6 @@
 """Crawlee loads lazily; the browser patch stays eager.
 
-Importing Crawlee costs ~1 s per run under gVisor, so ``import scrapely`` must
+Importing Crawlee costs ~1 s per run on the platform, so ``import scrapely`` must
 not import it. The browser patch must still be active right after import: the
 raw Playwright patch at once, the Crawlee BrowserPool / PlaywrightCrawler
 patches as soon as Crawlee's modules load.
@@ -292,3 +292,20 @@ def test_actor_module_lazy_names_are_patchable(monkeypatch: pytest.MonkeyPatch)
     sentinel = object()
     monkeypatch.setattr(actor_mod, 'Dataset', sentinel)
     assert actor_mod._lazy('Dataset') is sentinel
+
+
+def test_reference_crawlee_reexports_resolve_lazily() -> None:
+    """Every crawlee name the reference SDK re-exports is available, and only loads crawlee on access."""
+    result = _run(f"""
+        import json, sys
+        import scrapely
+        before = {_CRAWLEE_MODULES}
+        names = ['Request', 'Event', 'EventAbortingData', 'EventExitData', 'EventListener',
+                 'EventMigratingData', 'EventPersistStateData', 'EventSystemInfoData',
+                 'Dataset', 'KeyValueStore', 'RequestQueue']
+        import crawlee, crawlee.events
+        ok = {{n: getattr(scrapely, n) is (getattr(crawlee, n, None) or getattr(crawlee.events, n, None) or getattr(scrapely, n)) for n in names}}
+        print(json.dumps({{'before': before, 'ok': ok}}))
+    """)
+    assert result['before'] == []
+    assert all(result['ok'].values()), result['ok']

{scrapely_python-0.1.26 → scrapely_python-0.1.28}/tests/test_patch_browser_pool.py

@@ -1017,6 +1017,8 @@ async def test_pyppeteer_launch_requires_cdp_url_on_platform(pyppeteer_stub, mon
     pyppeteer_stub.connect.assert_not_awaited()
 
 
+_SAFE_ARGS = ['--disable-breakpad', '--disable-crash-reporter', '--disable-dev-shm-usage', '--crash-dumps-dir=/tmp']
+
 # ─── SCRAPELY_BROWSER_PATCH=0 (unpatched build: actor launches its own browsers) ─
 
 
@@ -1025,7 +1027,6 @@ def browser_patch_off(monkeypatch):
     """On the platform with a CDP URL, but the build opted out of the patch."""
     monkeypatch.setenv('SCRAPELY_IS_AT_HOME', '1')
     monkeypatch.setenv('SCRAPELY_BROWSER_PATCH', '0')
-    monkeypatch.delenv('SCRAPELY_CONTAINER_RUNTIME', raising=False)
     set_cdp_url('wss://browser.test/cdp')
 
 
@@ -1099,8 +1100,8 @@ def test_patch_off_raw_sync_launch_and_lpc_run_locally(browser_patch_off):
     p.firefox.launch(headless=True)
     p.chromium.launch_persistent_context('/tmp/profile', headless=True)
 
-    p.firefox.original_launch.assert_called_once_with(headless=True)
-    p.chromium.original_launch_persistent_context.assert_called_once_with('/tmp/profile', headless=True)
+    p.firefox.original_launch.assert_called_once_with(headless=True, args=_SAFE_ARGS)
+    p.chromium.original_launch_persistent_context.assert_called_once_with('/tmp/profile', headless=True, args=_SAFE_ARGS)
     p.chromium.connect_over_cdp.assert_not_called()
 
 
@@ -1111,7 +1112,7 @@ def test_patch_off_raw_launch_ignores_browser_configuration(browser_patch_off):
 
     p.chromium.launch(headless=True, browser_configuration=BrowserConfiguration(os='macos'))
 
-    p.chromium.original_launch.assert_called_once_with(headless=True)
+    p.chromium.original_launch.assert_called_once_with(headless=True, args=_SAFE_ARGS)
     p.chromium.connect_over_cdp.assert_not_called()
 
 
@@ -1134,7 +1135,7 @@ async def test_patch_off_pyppeteer_ignores_browser_configuration(browser_patch_o
     pbp._apply_pyppeteer_patch()
 
     await pkg.launch({'headless': True, 'browser_configuration': BrowserConfiguration(os='macos')})
-    original.assert_awaited_once_with({'headless': True})
+    original.assert_awaited_once_with({'headless': True, 'args': _SAFE_ARGS})
 
 
 @pytest.mark.asyncio
@@ -1172,12 +1173,11 @@ async def test_patch_off_raw_async_launch_runs_locally(browser_patch_off):
     pbp._patch_playwright_object(p, is_async=True)
 
     assert await p.chromium.launch(headless=True) == 'local-browser'
-    chromium.original_launch.assert_awaited_once_with(headless=True)
+    chromium.original_launch.assert_awaited_once_with(headless=True, args=_SAFE_ARGS)
     chromium.connect_over_cdp.assert_not_awaited()
 
 
-def test_patch_off_adds_gvisor_args_under_runsc(browser_patch_off, monkeypatch):
-    monkeypatch.setenv('SCRAPELY_CONTAINER_RUNTIME', 'runsc')
+def test_patch_off_adds_sandbox_args_on_platform(browser_patch_off, monkeypatch):
     p = _fake_sync_driver()
 
     p.chromium.launch(args=['--x'])
@@ -1206,7 +1206,7 @@ async def test_patch_off_pyppeteer_runs_original(browser_patch_off, monkeypatch)
     pbp._apply_pyppeteer_patch()
 
     assert await pkg.launch(headless=True, args=['--x']) == 'local-browser'
-    original.assert_awaited_once_with({'headless': True, 'args': ['--x']})
+    original.assert_awaited_once_with({'headless': True, 'args': ['--x', *_SAFE_ARGS]})
     connect.assert_not_awaited()