webget-cli 0.8.0__tar.gz → 0.10.0__tar.gz

{webget_cli-0.8.0/webget_cli.egg-info → webget_cli-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.8.0
+Version: 0.10.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -247,7 +247,7 @@ standalone with `webget-mcp` (stdio transport) or `python webget_mcp.py`.
 ### Authenticated sessions (profiles)
 
 MCP tools can use locally stored login sessions. Create one first with the
-CLI:
+CLI, or let the agent create it via the MCP `login` tool:
 
 ```bash
 webget login https://portal.example.com --profile portal
@@ -257,13 +257,19 @@ Then the agent can discover sessions and fetch authenticated pages:
 
 - `list_profiles` - lists available sessions (name, last used, size,
   status). Cookie values are never returned.
+- `login(url, profile)` - opens a browser session (headful by default so a
+  human can log in), navigates to `url`, and persists the session once the
+  login handshake's cookies appear (or after `wait_seconds`, whichever
+  comes first). MCP stdin is the JSON-RPC stream, so there is no Enter
+  keypress; the flow polls for cookies instead.
 - `fetch(..., profile="portal")` / `search_fetch(..., profile="portal")` -
   scrape using that session.
 
 ```text
 agent: "check my portal for new announcements"
   1. list_profiles            -> portal (authenticated)
-  2. fetch(url, profile="portal")
+  2. login(https://portal.example.com, profile="portal")  # if not listed
+  3. fetch(url, profile="portal")
 ```
 
 Invalid profile names and unknown profiles are hard errors (no silent

{webget_cli-0.8.0 → webget_cli-0.10.0}/README.md

@@ -213,7 +213,7 @@ standalone with `webget-mcp` (stdio transport) or `python webget_mcp.py`.
 ### Authenticated sessions (profiles)
 
 MCP tools can use locally stored login sessions. Create one first with the
-CLI:
+CLI, or let the agent create it via the MCP `login` tool:
 
 ```bash
 webget login https://portal.example.com --profile portal
@@ -223,13 +223,19 @@ Then the agent can discover sessions and fetch authenticated pages:
 
 - `list_profiles` - lists available sessions (name, last used, size,
   status). Cookie values are never returned.
+- `login(url, profile)` - opens a browser session (headful by default so a
+  human can log in), navigates to `url`, and persists the session once the
+  login handshake's cookies appear (or after `wait_seconds`, whichever
+  comes first). MCP stdin is the JSON-RPC stream, so there is no Enter
+  keypress; the flow polls for cookies instead.
 - `fetch(..., profile="portal")` / `search_fetch(..., profile="portal")` -
   scrape using that session.
 
 ```text
 agent: "check my portal for new announcements"
   1. list_profiles            -> portal (authenticated)
-  2. fetch(url, profile="portal")
+  2. login(https://portal.example.com, profile="portal")  # if not listed
+  3. fetch(url, profile="portal")
 ```
 
 Invalid profile names and unknown profiles are hard errors (no silent

{webget_cli-0.8.0 → webget_cli-0.10.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.8.0"
+version = "0.10.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -39,11 +39,12 @@ mcp = ["fastmcp>=2"]
 dev = ["pytest>=8", "ruff>=0.6"]
 
 [project.scripts]
-webget = "webget_cli:main"
+webget = "webget.cli:main"
 webget-mcp = "webget_mcp:main"
 
 [tool.setuptools]
 py-modules = ["webget_cli", "webget_mcp"]
+packages = ["webget"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

{webget_cli-0.8.0 → webget_cli-0.10.0}/tests/test_adversarial_http.py

@@ -1,6 +1,7 @@
 """Adversarial HTTP tests: URL handling, status codes, redirects, bodies."""
 
 import asyncio
+import time
 
 import pytest
 
@@ -63,6 +64,26 @@ class TestConnectionFailures:
         res = asyncio.run(_fetch(server.url("/timeout?sec=5"), per_url_timeout=1))
         assert _one(res)["status"] == "error"
 
+    def test_slow_drip_hits_wall_clock_deadline(self, fresh_cache):
+        """A server that slow-drips the body must not exceed per_url_timeout.
+
+        httpx's own timeout bounds a single socket operation only, so a
+        drip of small chunks every 150ms (total ~6s for 40 chunks) would
+        keep the request alive far past a 2s budget. fetch_http enforces
+        an absolute deadline while streaming; this proves the URL errors
+        out in roughly the budget, not the full drip duration.
+        """
+        server = fresh_cache
+        t0 = time.monotonic()
+        res = asyncio.run(
+            _fetch(server.url("/drip?n=40&delay=0.15"), per_url_timeout=2, strategy="http")
+        )
+        elapsed = time.monotonic() - t0
+        out = _one(res)
+        assert out["status"] == "error"
+        # Drip would take ~6s uncapped; deadline must cut it near 2s.
+        assert elapsed < 4.0, f"deadline not enforced: took {elapsed:.1f}s"
+
 
 # ---------- status codes ----------
 

{webget_cli-0.8.0 → webget_cli-0.10.0}/tests/test_adversarial_ssrf.py

@@ -98,14 +98,14 @@ class TestSSRFBlocking:
         the initial URL passed the guard (e.g. it was public)."""
         import pytest
 
-        real = webget._is_private_target
+        real = webget._private_ip_for
         initial = server.url("/redirect-private")
 
         def guarded(url):
             # initial URL passes (as if public); every other URL checked for real
-            return real(url) if url != initial else False
+            return real(url) if url != initial else None
 
-        monkeypatch.setattr(webget, "_is_private_target", guarded)
+        monkeypatch.setattr(webget, "_private_ip_for", guarded)
         with pytest.raises(webget.SSRFError):
             asyncio.run(webget.fetch_http(initial, 2000, timeout=5))
 

webget_cli-0.10.0/tests/test_login_flow.py

@@ -0,0 +1,180 @@
+"""MCP login tool + non-interactive login flow tests.
+
+Covers:
+  - _login_flow(interactive=False): polls browser context until a Set-Cookie
+    arrives (here: /set-cookie on the local test server), then persists a
+    storage_state.json the HTTP fast path can reuse.
+  - MCP login tool: validates URL/profile, runs the flow headless against
+    the local server, then fetch(profile=...) uses the stored session.
+  - login tool rejects bad URLs and unknown/invalid profiles.
+
+The test server lives on 127.0.0.1, so WEBGET_ALLOW_PRIVATE=1 is required
+for the subsequent authenticated fetch through the SSRF-guarded HTTP path.
+"""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+import webget_cli as webget
+
+playwright = pytest.importorskip("playwright", reason="playwright not installed")
+
+
+def _profile_root(tmp_path):
+    return tmp_path / "profiles"
+
+
+def test_login_flow_noninteractive_persists_session(server, tmp_path, monkeypatch):
+    """Browser-less-enough path: headless Chromium navigates /set-cookie,
+    cookie polling sees the Set-Cookie side effect, storage_state persists,
+    and the HTTP fast path can reuse it on /cookie-gated."""
+    root = _profile_root(tmp_path)
+    monkeypatch.setattr(webget, "PROFILE_DIR", str(root))
+    monkeypatch.setenv("WEBGET_ALLOW_PRIVATE", "1")
+
+    asyncio.run(
+        webget._login_flow(
+            server.url("/set-cookie"),
+            "flowp",
+            headless=True,
+            wait_seconds=30,
+            interactive=False,
+            quiet=True,
+        )
+    )
+
+    state_p = webget.profile_state_path("flowp")
+    assert os.path.exists(state_p)
+    state = json.loads(Path(state_p).read_text())
+    cookies = {c["name"]: c["value"] for c in state["cookies"]}
+    assert cookies.get("webget_session") == "1"
+
+    # HTTP fast path must now authenticate on the cookie-gated page.
+    res = asyncio.run(
+        webget.scrape_many(
+            [server.url("/cookie-gated")],
+            strategy="http",
+            no_cache=True,
+            profile="flowp",
+        )
+    )
+    out = res[server.url("/cookie-gated")]
+    assert out["status"] == "success"
+    assert out["auth"]["authenticated"] is True
+    assert "you are authenticated" in out["markdown"]
+
+
+def test_login_flow_waits_until_cookie(server, tmp_path, monkeypatch):
+    """A page that does NOT set a cookie still persists (best-effort), and a
+    slow Set-Cookie is picked up by polling before the deadline."""
+    root = _profile_root(tmp_path)
+    monkeypatch.setattr(webget, "PROFILE_DIR", str(root))
+
+    asyncio.run(
+        webget._login_flow(
+            server.url("/set-cookie"),
+            "slowp",
+            headless=True,
+            wait_seconds=30,
+            interactive=False,
+            quiet=True,
+        )
+    )
+    state_p = webget.profile_state_path("slowp")
+    assert os.path.exists(state_p)
+    cookies = {c["name"] for c in json.loads(Path(state_p).read_text())["cookies"]}
+    assert "webget_session" in cookies
+
+
+def _spawn_mcp(profile_root):
+    from mcp import StdioServerParameters
+
+    env = {
+        **os.environ,
+        "WEBGET_PROFILE_DIR": str(profile_root),
+        "WEBGET_ALLOW_PRIVATE": "1",
+    }
+    return StdioServerParameters(command=sys.executable, args=[str(ROOT / "webget_mcp.py")], env=env)
+
+
+def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
+    """End-to-end MCP: login(url, profile, headless=True) against the local
+    /set-cookie page persists a session, then fetch(profile=...) on the
+    cookie-gated page succeeds while anonymous fetch does not."""
+    from mcp import ClientSession
+    from mcp.client.stdio import stdio_client
+
+    root = _profile_root(tmp_path)
+    login_url = server.url("/set-cookie")
+    gated_url = server.url("/cookie-gated")
+
+    async def run():
+        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "login",
+                {"url": login_url, "profile": "mcplogin", "headless": True, "wait_seconds": 30},
+            )
+            authed = await session.call_tool(
+                "fetch", {"url": gated_url, "strategy": "http", "no_cache": True, "profile": "mcplogin"}
+            )
+            anon = await session.call_tool(
+                "fetch", {"url": gated_url, "strategy": "http", "no_cache": True}
+            )
+            return res, authed, anon
+
+    res, authed, anon = asyncio.run(asyncio.wait_for(run(), timeout=90))
+    res_text = res.content[0].text if res.content else ""
+    assert not res.isError, res_text
+    assert '"status":"success"' in res_text, res_text
+    assert '"profile":"mcplogin"' in res_text
+
+    authed_text = authed.content[0].text if authed.content else ""
+    assert '"status":"success"' in authed_text, authed_text
+    assert "you are authenticated" in authed_text
+    assert '"authenticated":true' in authed_text
+
+    anon_text = anon.content[0].text if anon.content else ""
+    assert '"status":"success"' not in anon_text, f"anonymous fetch leaked: {anon_text}"
+
+
+def test_mcp_login_rejects_bad_url_and_profile(server, tmp_path):
+    """login tool validates its inputs: bad URL and bad profile name are
+    clean errors, never a crash."""
+    from mcp import ClientSession
+    from mcp.client.stdio import stdio_client
+
+    root = _profile_root(tmp_path)
+
+    async def run():
+        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            bad_url = await session.call_tool(
+                "login", {"url": "not a url", "profile": "x", "headless": True, "wait_seconds": 5}
+            )
+            bad_name = await session.call_tool(
+                "login", {"url": server.url("/"), "profile": "../evil", "headless": True, "wait_seconds": 5}
+            )
+            bad_secs = await session.call_tool(
+                "login", {"url": server.url("/"), "profile": "x", "headless": True, "wait_seconds": 1}
+            )
+            return bad_url, bad_name, bad_secs
+
+    bad_url, bad_name, bad_secs = asyncio.run(asyncio.wait_for(run(), timeout=60))
+    assert "invalid site URL" in bad_url.content[0].text
+    assert "invalid profile name" in bad_name.content[0].text
+    assert "wait_seconds must be between 5 and 600" in bad_secs.content[0].text
+
+
+if __name__ == "__main__":
+    import pytest
+
+    sys.exit(pytest.main([__file__, "-v"]))

{webget_cli-0.8.0 → webget_cli-0.10.0}/tests/test_mcp_profile.py

@@ -43,7 +43,7 @@ def _make_profile(root, name):
             {
                 "cookies": [
                     {
-                        "name": "s",
+                        "name": "session",
                         "value": "1",
                         "domain": "127.0.0.1",
                         "path": "/",
@@ -109,6 +109,68 @@ def test_mcp_fetch_unknown_profile_is_hard_error(tmp_path):
     assert "profile 'ghost' not found" in res.content[0].text
 
 
+def test_mcp_fetch_enriched_output_present(tmp_path):
+    """MCP fetch must include auth, attempts, and reasons in the response.
+
+    These fields give agents full provenance: whether the fetch used a
+    session, how many ladder steps were tried, and the full chain of
+    failures. Added 2026-08-31.
+    """
+    server = TestServer().start()
+    try:
+        root = tmp_path / "profiles"
+        _make_profile(root, "testp")
+        gated = server.url("/cookie-gated")
+
+        async def run():
+            async with (
+                stdio_client(_spawn(root)) as (read, write),
+                ClientSession(read, write) as session,
+            ):
+                await session.initialize()
+                # Authenticated fetch
+                ok = await session.call_tool(
+                    "fetch",
+                    {
+                        "url": gated,
+                        "strategy": "http",
+                        "no_cache": True,
+                        "profile": "testp",
+                    },
+                )
+                # Anonymous fetch (blocked)
+                anon = await session.call_tool(
+                    "fetch",
+                    {"url": gated, "strategy": "http", "no_cache": True},
+                )
+                return ok, anon
+
+        ok, anon = asyncio.run(asyncio.wait_for(run(), timeout=60))
+        ok_text = ok.content[0].text if ok.content else ""
+        anon_text = anon.content[0].text if anon.content else ""
+
+        # Authenticated: standard assertions
+        assert not ok.isError, ok_text
+        assert '"status":"success"' in ok_text
+        # auth
+        assert '"auth"' in ok_text
+        assert '"authenticated":true' in ok_text
+        assert '"attempts"' in ok_text
+        # Success carries an empty reasons list (consistent shape: consumers
+        # can always iterate reasons without a None check).
+        assert '"reasons":[]' in ok_text, "success should have empty reasons"
+
+        # Anonymous (blocked): must have provenance fields
+        assert '"status":"blocked"' in anon_text or '"status":"login_required"' in anon_text
+        assert '"auth"' in anon_text
+        assert '"authenticated":false' in anon_text or '"authenticated":null' in anon_text
+        assert '"attempts"' in anon_text
+        assert '"reasons"' in anon_text
+        assert '"state":"blocked"' in anon_text or '"state":"login_required"' in anon_text
+    finally:
+        server.stop()
+
+
 if __name__ == "__main__":
     import pytest
 

{webget_cli-0.8.0 → webget_cli-0.10.0}/tests/test_mcp_server.py

@@ -30,7 +30,9 @@ def test_tools_listed():
             return [t.name for t in tools.tools]
 
     # order is not a contract; membership is
-    assert sorted(_run(run())) == sorted(["search", "fetch", "search_fetch", "list_profiles"])
+    assert sorted(_run(run())) == sorted(
+        ["search", "fetch", "search_fetch", "list_profiles", "login"]
+    )
 
 
 def test_invalid_strategy_returns_error_not_crash():

{webget_cli-0.8.0 → webget_cli-0.10.0}/tests/test_security_review.py

@@ -83,30 +83,30 @@ class TestRedirectHopPolicy:
         hop check must fire even though the initial URL passed."""
         import pytest
 
-        real = webget._is_private_target
+        real = webget._private_ip_for
         initial = server.url("/redirect-private")
 
         def guarded(url):
-            return real(url) if url != initial else False
+            return real(url) if url != initial else None
 
-        monkeypatch.setattr(webget, "_is_private_target", guarded)
+        monkeypatch.setattr(webget, "_private_ip_for", guarded)
         with pytest.raises(webget.SSRFError):
             asyncio.run(webget.fetch_http(initial, 2000, timeout=5))
 
     def test_multi_hop_into_private_blocked(self, server, isolated_env, monkeypatch):
         """public -> public -> private (3 hops) must still be blocked."""
 
-        real = webget._is_private_target
+        real = webget._private_ip_for
         initial = server.url("/redirect-chain?n=2")  # -> /redirect-chain?n=1 -> /normal
         # craft: first two hops pass (public), third hop lands on /private
         chain = [server.url("/redirect-chain?n=2"), server.url("/redirect-chain?n=1")]
 
         def guarded(url):
             if url in chain or url == server.url("/redirect-chain?n=2"):
-                return False
+                return None
             return real(url)
 
-        monkeypatch.setattr(webget, "_is_private_target", guarded)
+        monkeypatch.setattr(webget, "_private_ip_for", guarded)
         # fetch_http follows hops manually; every hop re-checks
         try:
             asyncio.run(webget.fetch_http(initial, 2000, timeout=5))

webget_cli-0.10.0/tests/test_size_review.py

@@ -0,0 +1,109 @@
+"""Phase 11 response-size-limit review tests.
+
+The cap must be enforced WHILE STREAMING (client.stream), not after the
+body has been buffered. /oversize serves 30MB; a non-streaming client
+would buffer all 30MB before the check could fire.
+"""
+
+import asyncio
+import time
+
+import webget_cli as webget
+
+
+async def _fetch(url, **kw):
+    # Default to explicit http strategy: these tests measure the HTTP
+    # streaming cap, not the ladder. With strategy="auto" a too-large
+    # response would escalate to Crawl4AI (browser import ~40s), which
+    # pollutes the wall-time assertion.
+    kw.setdefault("strategy", "http")
+    return await webget.scrape_many([url], no_cache=True, **kw)
+
+
+def _one(res):
+    return res[next(iter(res))]
+
+
+class TestSizeLimitEnforcement:
+    def test_oversize_returns_error(self, fresh_cache):
+        server = fresh_cache
+        res = asyncio.run(_fetch(server.url("/oversize")))
+        out = _one(res)
+        assert out["status"] == "error"
+        assert "too large" in (out.get("error") or "").lower()
+
+    def test_undersize_still_works(self, fresh_cache):
+        server = fresh_cache
+        res = asyncio.run(_fetch(server.url("/huge")))
+        assert _one(res)["status"] == "success"
+
+    def test_streaming_actually_bounded(self, fresh_cache):
+        """The cap MUST abort mid-stream, not buffer the full 30MB.
+
+        Verified by wall time: if the client buffered 30MB over localhost
+        the wall time would be under 1s (fast local pipe).  But the
+        streaming cap fires at 25MB, so the function returns quickly
+        with an error -- well before a full 30MB read + extraction.
+        The maxrss measurement is unreliable here because the test
+        server runs in-process (same memory space), so we rely on
+        wall time as a proxy: a full 30MB read + extraction would take
+        noticeably longer.
+        """
+        server = fresh_cache
+        t0 = time.perf_counter()
+        res = asyncio.run(_fetch(server.url("/oversize")))
+        wall = time.perf_counter() - t0
+        out = _one(res)
+        assert out["status"] == "error"
+        assert "too large" in (out.get("error") or "").lower()
+        # Local server: 25MB cap should fire in < 5s. If the cap broke
+        # and the client read all 30MB, extraction adds significant time.
+        assert wall < 15, f"oversize fetch took {wall}s (looks like full read)"
+
+    def test_missing_content_length(self, fresh_cache):
+        """Chunked/unknown-length responses must still hit the cap."""
+        # /oversize declares Content-Length, but the cap check does not
+        # depend on it; a chunked server would behave the same. This test
+        # documents the contract: cap is on bytes read, not on headers.
+        server = fresh_cache
+        res = asyncio.run(_fetch(server.url("/oversize")))
+        assert _one(res)["status"] == "error"
+
+    def test_too_large_is_terminal_not_escalated(self, fresh_cache, monkeypatch):
+        """ResponseTooLarge must NOT trigger the crawl4ai ladder step.
+
+        Before this fix, any http exception (including the streaming cap)
+        returned None from record(), leaving the URL pending, so the
+        ladder imported crawl4ai and launched a browser to re-download
+        the same 30MB body. The cap is terminal: report the error, drop
+        the URL, never open a browser.
+        """
+        server = fresh_cache
+        # The test server's /oversize is on 127.0.0.1 (private), so allow it.
+        monkeypatch.setenv("WEBGET_ALLOW_PRIVATE", "1")
+        res = asyncio.run(_fetch(server.url("/oversize")))
+        out = _one(res)
+        # Terminal: status=error from the http step, not escalated.
+        assert out["status"] == "error"
+        assert out["method"] == "http"
+        assert "too large" in (out.get("error") or "").lower()
+        # It must resolve in one step (no browser re-download), so the
+        # wall time stays low. (The ladder would have imported crawl4ai
+        # and launched Chromium otherwise, which is far slower.)
+        assert out["attempts"] == 1
+
+
+class TestBrowserEquivalent:
+    def test_browser_has_no_byte_cap_documented(self):
+        """The browser strategy loads full pages (JS rendering) and has no
+        equivalent byte cap: crawl4ai returns whatever Chromium parsed.
+        This is a documented difference, not a silent gap: page size is
+        bounded in practice by Chromium's own resource limits and by
+        max_chars truncation AFTER parsing. A 30MB page would still be
+        parsed by the browser. Acceptable risk for the browser path."""
+        import inspect
+
+        src = inspect.getsource(webget.scrape_many)
+        assert "MAX_RESPONSE_BYTES" in inspect.getsource(webget)  # exists
+        # The browser path intentionally relies on crawl4ai's own handling.
+        assert "crawl4ai" in src