webget-cli 0.7.1__tar.gz → 0.7.2__tar.gz

{webget_cli-0.7.1 → webget_cli-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.7.1
+Version: 0.7.2
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: MIT

{webget_cli-0.7.1 → webget_cli-0.7.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.7.1"
+version = "0.7.2"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"

{webget_cli-0.7.1 → webget_cli-0.7.2}/tests/test_adversarial_ssrf.py

@@ -130,3 +130,41 @@ class TestSSRFGuardLadder:
         out = _one(res)
         assert "attempts" in out
         assert out["method"] in ("http", "crawl4ai", "firecrawl", "")
+
+
+class TestRequestBodyBytes:
+    """Regression (0.7.1 bug): the browser route guard read request.post_data,
+    which decodes UTF-8 and raises UnicodeDecodeError on binary/compressed
+    bodies (gzip POSTs, seen on facebook/linkedin). The guard must read the
+    UNDECODED bytes (post_data_buffer), and body handling must never be part
+    of the SSRF decision."""
+
+    class _BinaryRequest:
+        """Stub of a Playwright request with a gzip/binary body."""
+
+        @property
+        def post_data(self):
+            # The exact 0.7.1 crash: post_data forces UTF-8 decode.
+            raise UnicodeDecodeError("utf-8", b"\x8b", 0, 1, "invalid start byte")
+
+        @property
+        def post_data_buffer(self):
+            return b"\x1f\x8b\x08\x00binary-gzip-body"
+
+    def test_returns_undecoded_bytes(self):
+        assert webget._request_body_bytes(self._BinaryRequest()) == (
+            b"\x1f\x8b\x08\x00binary-gzip-body"
+        )
+
+    def test_never_calls_post_data(self):
+        """If someone reverts to request.post_data, this test must fail."""
+        r = self._BinaryRequest()
+        assert webget._request_body_bytes(r) is not None  # would raise otherwise
+
+    def test_none_on_any_error(self):
+        class _Broken:
+            @property
+            def post_data_buffer(self):
+                raise RuntimeError("transport closed")
+
+        assert webget._request_body_bytes(_Broken()) is None

{webget_cli-0.7.1 → webget_cli-0.7.2}/tests/test_browser_ssrf.py

@@ -16,6 +16,7 @@ request at all, not merely that its response was discarded.
 """
 
 import asyncio
+import gzip
 import sys
 from pathlib import Path
 
@@ -147,3 +148,58 @@ class TestNavigationRedirectSSRF:
         assert out["status"] == "error"
         # scrape_many's pre-check blocks before the browser ever runs.
         assert server.hits_for(PRIVATE_PAGE) == 0
+
+
+class TestBinaryPostGuard:
+    """Regression (0.7.1 bug): the route guard read request.post_data which
+    decodes UTF-8 and raises UnicodeDecodeError on gzip/binary POST bodies
+    (seen on facebook/linkedin). The handler died BEFORE the private-address
+    check, Playwright continued the request natively, and a binary POST to a
+    private target BYPASSED the SSRF guard entirely.
+
+    Fix: guard reads undecoded bytes (post_data_buffer); SSRF decisions are
+    URL/IP policy only. These tests prove:
+      - binary POST to PUBLIC target: guard alive, exact body replayed
+      - binary POST to PRIVATE target: aborted, endpoint NEVER receives it
+    """
+
+    BINARY_BODY = gzip.compress(b"binary payload " * 50)
+
+    def _driver_url(self, server, target):
+        return server.url(f"/binary-post-driver?to={target}")
+
+    def test_binary_post_to_public_allowed(self, isolated, monkeypatch):
+        server = isolated
+        driver = "/binary-post-driver?to=/binary-post"
+        monkeypatch.setattr(
+            webget,
+            "_is_private_target",
+            _fake_public_policy(server, extra_public=(driver, "/binary-post")),
+        )
+        server.reset_counters()
+        res = asyncio.run(_browser_fetch(server.url(driver)))
+        out = res[server.url(driver)]
+        assert out["status"] == "success", out
+        # The gzip POST went through the guard; the server received the
+        # EXACT bytes (guard replayed them via route.fetch, no crash).
+        assert server.hits_for("/binary-post") == 1
+        assert server.last_body("/binary-post") == self.BINARY_BODY
+
+    def test_binary_post_to_private_blocked(self, isolated, monkeypatch):
+        server = isolated
+        driver = "/binary-post-driver?to=/binary-post-private"
+        monkeypatch.setattr(
+            webget,
+            "_is_private_target",
+            _fake_public_policy(server, extra_public=(driver,)),
+        )
+        server.reset_counters()
+        res = asyncio.run(_browser_fetch(server.url(driver)))
+        out = res[server.url(driver)]
+        assert out["status"] == "success", out  # driver page itself is fine
+        # The private sink must NEVER receive the binary POST: with the
+        # 0.7.1 bug the guard crashed before the private check and the
+        # request leaked through (native continue). Zero hits = no bypass.
+        assert server.hits_for("/binary-post-private") == 0, (
+            f"private binary POST leaked {server.hits_for('/binary-post-private')} request(s)"
+        )

{webget_cli-0.7.1 → webget_cli-0.7.2}/webget_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.7.1
+Version: 0.7.2
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: MIT

{webget_cli-0.7.1 → webget_cli-0.7.2}/webget_cli.py

@@ -414,6 +414,22 @@ class SSRFError(RuntimeError):
     """Raised when a fetch would target a private address."""
 
 
+def _request_body_bytes(request):
+    """Raw request body as bytes for redirect replay, or None.
+
+    Playwright's `request.post_data` decodes the body as UTF-8 text and
+    raises UnicodeDecodeError on binary/compressed bodies (e.g. gzip
+    POSTs, seen on facebook/linkedin). The browser SSRF route guard
+    replays the raw body on manual redirect hops, so it must read the
+    undecoded bytes. Body handling is BEST-EFFORT and must never be part
+    of the SSRF decision, which is URL/IP policy only.
+    """
+    try:
+        return request.post_data_buffer
+    except Exception:  # noqa: BLE001 - body replay is best-effort
+        return None
+
+
 async def _guard_browser_routes(crawler_ctx):
     """Install a Playwright route guard so the browser can never request a
     private address, even through redirects or subresources.
@@ -443,7 +459,9 @@ async def _guard_browser_routes(crawler_ctx):
         # method/body handling: 301/302/303 upgrades redirects to GET per
         # HTTP spec; 307/308 preserve method+body.
         method = request.method
-        body = request.post_data
+        # post_data decodes as UTF-8 and raises on binary/compressed
+        # bodies; read the undecoded bytes instead (best-effort replay).
+        body = _request_body_bytes(request)
 
         for _hop in range(21):
             if _is_private_target(url):