webget-cli 0.7.3__tar.gz → 0.9.0__tar.gz

{webget_cli-0.7.3/webget_cli.egg-info → webget_cli-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.7.3
+Version: 0.9.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -244,9 +244,37 @@ Register as a local MCP server in opencode:
 Then prompt with `use webget` for search and scrape tasks. Run the server
 standalone with `webget-mcp` (stdio transport) or `python webget_mcp.py`.
 
-> **Limitation:** MCP tools do not expose `--profile`/`--cookies`, so
-> authenticated pages are out of scope for the MCP server. Use the CLI
-> (`webget login`, `webget u --profile ...`) for session-based fetching.
+### Authenticated sessions (profiles)
+
+MCP tools can use locally stored login sessions. Create one first with the
+CLI, or let the agent create it via the MCP `login` tool:
+
+```bash
+webget login https://portal.example.com --profile portal
+```
+
+Then the agent can discover sessions and fetch authenticated pages:
+
+- `list_profiles` - lists available sessions (name, last used, size,
+  status). Cookie values are never returned.
+- `login(url, profile)` - opens a browser session (headful by default so a
+  human can log in), navigates to `url`, and persists the session once the
+  login handshake's cookies appear (or after `wait_seconds`, whichever
+  comes first). MCP stdin is the JSON-RPC stream, so there is no Enter
+  keypress; the flow polls for cookies instead.
+- `fetch(..., profile="portal")` / `search_fetch(..., profile="portal")` -
+  scrape using that session.
+
+```text
+agent: "check my portal for new announcements"
+  1. list_profiles            -> portal (authenticated)
+  2. login(https://portal.example.com, profile="portal")  # if not listed
+  3. fetch(url, profile="portal")
+```
+
+Invalid profile names and unknown profiles are hard errors (no silent
+anonymous fallback). Sessions are local to the machine running the MCP
+server.
 
 ## Development
 

{webget_cli-0.7.3 → webget_cli-0.9.0}/README.md

@@ -210,9 +210,37 @@ Register as a local MCP server in opencode:
 Then prompt with `use webget` for search and scrape tasks. Run the server
 standalone with `webget-mcp` (stdio transport) or `python webget_mcp.py`.
 
-> **Limitation:** MCP tools do not expose `--profile`/`--cookies`, so
-> authenticated pages are out of scope for the MCP server. Use the CLI
-> (`webget login`, `webget u --profile ...`) for session-based fetching.
+### Authenticated sessions (profiles)
+
+MCP tools can use locally stored login sessions. Create one first with the
+CLI, or let the agent create it via the MCP `login` tool:
+
+```bash
+webget login https://portal.example.com --profile portal
+```
+
+Then the agent can discover sessions and fetch authenticated pages:
+
+- `list_profiles` - lists available sessions (name, last used, size,
+  status). Cookie values are never returned.
+- `login(url, profile)` - opens a browser session (headful by default so a
+  human can log in), navigates to `url`, and persists the session once the
+  login handshake's cookies appear (or after `wait_seconds`, whichever
+  comes first). MCP stdin is the JSON-RPC stream, so there is no Enter
+  keypress; the flow polls for cookies instead.
+- `fetch(..., profile="portal")` / `search_fetch(..., profile="portal")` -
+  scrape using that session.
+
+```text
+agent: "check my portal for new announcements"
+  1. list_profiles            -> portal (authenticated)
+  2. login(https://portal.example.com, profile="portal")  # if not listed
+  3. fetch(url, profile="portal")
+```
+
+Invalid profile names and unknown profiles are hard errors (no silent
+anonymous fallback). Sessions are local to the machine running the MCP
+server.
 
 ## Development
 

{webget_cli-0.7.3 → webget_cli-0.9.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.7.3"
+version = "0.9.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"

{webget_cli-0.7.3 → webget_cli-0.9.0}/tests/test_adversarial_http.py

@@ -1,6 +1,7 @@
 """Adversarial HTTP tests: URL handling, status codes, redirects, bodies."""
 
 import asyncio
+import time
 
 import pytest
 
@@ -63,6 +64,26 @@ class TestConnectionFailures:
         res = asyncio.run(_fetch(server.url("/timeout?sec=5"), per_url_timeout=1))
         assert _one(res)["status"] == "error"
 
+    def test_slow_drip_hits_wall_clock_deadline(self, fresh_cache):
+        """A server that slow-drips the body must not exceed per_url_timeout.
+
+        httpx's own timeout bounds a single socket operation only, so a
+        drip of small chunks every 150ms (total ~6s for 40 chunks) would
+        keep the request alive far past a 2s budget. fetch_http enforces
+        an absolute deadline while streaming; this proves the URL errors
+        out in roughly the budget, not the full drip duration.
+        """
+        server = fresh_cache
+        t0 = time.monotonic()
+        res = asyncio.run(
+            _fetch(server.url("/drip?n=40&delay=0.15"), per_url_timeout=2, strategy="http")
+        )
+        elapsed = time.monotonic() - t0
+        out = _one(res)
+        assert out["status"] == "error"
+        # Drip would take ~6s uncapped; deadline must cut it near 2s.
+        assert elapsed < 4.0, f"deadline not enforced: took {elapsed:.1f}s"
+
 
 # ---------- status codes ----------
 

{webget_cli-0.7.3 → webget_cli-0.9.0}/tests/test_adversarial_mcp.py

@@ -58,7 +58,7 @@ class TestMalformedArguments:
                 return res
 
         res = _run(run())
-        assert res.isError
+        assert res.is_error
 
 
 class TestInvalidURLs:
@@ -72,7 +72,7 @@ class TestInvalidURLs:
                 return res
 
         res = _run(run())
-        assert "error" in res.content[0].text or res.isError
+        assert "error" in res.content[0].text or res.is_error
 
 
 class TestRepeatedCalls:
@@ -86,7 +86,7 @@ class TestRepeatedCalls:
                         "fetch",
                         {"url": "https://example.com", "strategy": "http", "no_cache": True},
                     )
-                    if res.isError:
+                    if res.is_error:
                         return "ERROR"
                 return "OK"
 
@@ -106,7 +106,7 @@ class TestRepeatedCalls:
                         for _ in range(5)
                     ]
                 )
-                return [r.isError for r in results]
+                return [r.is_error for r in results]
 
         assert _run(run()) == [False] * 5
 
@@ -133,7 +133,7 @@ class TestInputCaps:
                 return res
 
         res = _run(run())
-        assert "must be between" in res.content[0].text or res.isError
+        assert "must be between" in res.content[0].text or res.is_error
 
 
 class TestToolFailureIsolation:

{webget_cli-0.7.3 → webget_cli-0.9.0}/tests/test_adversarial_ssrf.py

@@ -98,14 +98,14 @@ class TestSSRFBlocking:
         the initial URL passed the guard (e.g. it was public)."""
         import pytest
 
-        real = webget._is_private_target
+        real = webget._private_ip_for
         initial = server.url("/redirect-private")
 
         def guarded(url):
             # initial URL passes (as if public); every other URL checked for real
-            return real(url) if url != initial else False
+            return real(url) if url != initial else None
 
-        monkeypatch.setattr(webget, "_is_private_target", guarded)
+        monkeypatch.setattr(webget, "_private_ip_for", guarded)
         with pytest.raises(webget.SSRFError):
             asyncio.run(webget.fetch_http(initial, 2000, timeout=5))
 

webget_cli-0.9.0/tests/test_login_flow.py

@@ -0,0 +1,180 @@
+"""MCP login tool + non-interactive login flow tests.
+
+Covers:
+  - _login_flow(interactive=False): polls browser context until a Set-Cookie
+    arrives (here: /set-cookie on the local test server), then persists a
+    storage_state.json the HTTP fast path can reuse.
+  - MCP login tool: validates URL/profile, runs the flow headless against
+    the local server, then fetch(profile=...) uses the stored session.
+  - login tool rejects bad URLs and unknown/invalid profiles.
+
+The test server lives on 127.0.0.1, so WEBGET_ALLOW_PRIVATE=1 is required
+for the subsequent authenticated fetch through the SSRF-guarded HTTP path.
+"""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+import webget_cli as webget
+
+playwright = pytest.importorskip("playwright", reason="playwright not installed")
+
+
+def _profile_root(tmp_path):
+    return tmp_path / "profiles"
+
+
+def test_login_flow_noninteractive_persists_session(server, tmp_path, monkeypatch):
+    """Browser-less-enough path: headless Chromium navigates /set-cookie,
+    cookie polling sees the Set-Cookie side effect, storage_state persists,
+    and the HTTP fast path can reuse it on /cookie-gated."""
+    root = _profile_root(tmp_path)
+    monkeypatch.setattr(webget, "PROFILE_DIR", str(root))
+    monkeypatch.setenv("WEBGET_ALLOW_PRIVATE", "1")
+
+    asyncio.run(
+        webget._login_flow(
+            server.url("/set-cookie"),
+            "flowp",
+            headless=True,
+            wait_seconds=30,
+            interactive=False,
+            quiet=True,
+        )
+    )
+
+    state_p = webget.profile_state_path("flowp")
+    assert os.path.exists(state_p)
+    state = json.loads(Path(state_p).read_text())
+    cookies = {c["name"]: c["value"] for c in state["cookies"]}
+    assert cookies.get("webget_session") == "1"
+
+    # HTTP fast path must now authenticate on the cookie-gated page.
+    res = asyncio.run(
+        webget.scrape_many(
+            [server.url("/cookie-gated")],
+            strategy="http",
+            no_cache=True,
+            profile="flowp",
+        )
+    )
+    out = res[server.url("/cookie-gated")]
+    assert out["status"] == "success"
+    assert out["auth"]["authenticated"] is True
+    assert "you are authenticated" in out["markdown"]
+
+
+def test_login_flow_waits_until_cookie(server, tmp_path, monkeypatch):
+    """A page that does NOT set a cookie still persists (best-effort), and a
+    slow Set-Cookie is picked up by polling before the deadline."""
+    root = _profile_root(tmp_path)
+    monkeypatch.setattr(webget, "PROFILE_DIR", str(root))
+
+    asyncio.run(
+        webget._login_flow(
+            server.url("/set-cookie"),
+            "slowp",
+            headless=True,
+            wait_seconds=30,
+            interactive=False,
+            quiet=True,
+        )
+    )
+    state_p = webget.profile_state_path("slowp")
+    assert os.path.exists(state_p)
+    cookies = {c["name"] for c in json.loads(Path(state_p).read_text())["cookies"]}
+    assert "webget_session" in cookies
+
+
+def _spawn_mcp(profile_root):
+    from mcp import StdioServerParameters
+
+    env = {
+        **os.environ,
+        "WEBGET_PROFILE_DIR": str(profile_root),
+        "WEBGET_ALLOW_PRIVATE": "1",
+    }
+    return StdioServerParameters(command=sys.executable, args=[str(ROOT / "webget_mcp.py")], env=env)
+
+
+def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
+    """End-to-end MCP: login(url, profile, headless=True) against the local
+    /set-cookie page persists a session, then fetch(profile=...) on the
+    cookie-gated page succeeds while anonymous fetch does not."""
+    from mcp import ClientSession
+    from mcp.client.stdio import stdio_client
+
+    root = _profile_root(tmp_path)
+    login_url = server.url("/set-cookie")
+    gated_url = server.url("/cookie-gated")
+
+    async def run():
+        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "login",
+                {"url": login_url, "profile": "mcplogin", "headless": True, "wait_seconds": 30},
+            )
+            authed = await session.call_tool(
+                "fetch", {"url": gated_url, "strategy": "http", "no_cache": True, "profile": "mcplogin"}
+            )
+            anon = await session.call_tool(
+                "fetch", {"url": gated_url, "strategy": "http", "no_cache": True}
+            )
+            return res, authed, anon
+
+    res, authed, anon = asyncio.run(asyncio.wait_for(run(), timeout=90))
+    res_text = res.content[0].text if res.content else ""
+    assert not res.is_error, res_text
+    assert '"status":"success"' in res_text, res_text
+    assert '"profile":"mcplogin"' in res_text
+
+    authed_text = authed.content[0].text if authed.content else ""
+    assert '"status":"success"' in authed_text, authed_text
+    assert "you are authenticated" in authed_text
+    assert '"authenticated":true' in authed_text
+
+    anon_text = anon.content[0].text if anon.content else ""
+    assert '"status":"success"' not in anon_text, f"anonymous fetch leaked: {anon_text}"
+
+
+def test_mcp_login_rejects_bad_url_and_profile(server, tmp_path):
+    """login tool validates its inputs: bad URL and bad profile name are
+    clean errors, never a crash."""
+    from mcp import ClientSession
+    from mcp.client.stdio import stdio_client
+
+    root = _profile_root(tmp_path)
+
+    async def run():
+        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            bad_url = await session.call_tool(
+                "login", {"url": "not a url", "profile": "x", "headless": True, "wait_seconds": 5}
+            )
+            bad_name = await session.call_tool(
+                "login", {"url": server.url("/"), "profile": "../evil", "headless": True, "wait_seconds": 5}
+            )
+            bad_secs = await session.call_tool(
+                "login", {"url": server.url("/"), "profile": "x", "headless": True, "wait_seconds": 1}
+            )
+            return bad_url, bad_name, bad_secs
+
+    bad_url, bad_name, bad_secs = asyncio.run(asyncio.wait_for(run(), timeout=60))
+    assert "invalid site URL" in bad_url.content[0].text
+    assert "invalid profile name" in bad_name.content[0].text
+    assert "wait_seconds must be between 5 and 600" in bad_secs.content[0].text
+
+
+if __name__ == "__main__":
+    import pytest
+
+    sys.exit(pytest.main([__file__, "-v"]))

{webget_cli-0.7.3 → webget_cli-0.9.0}/tests/test_mcp_leak_review.py

@@ -64,7 +64,7 @@ class TestNoSecretLeakage:
                 return res
 
         res = _run(run())
-        if res.isError:
+        if res.is_error:
             return  # error path: no payload to leak, still fine
         text = res.content[0].text
         hits = _scan(text)
@@ -121,7 +121,7 @@ class TestServerRecovery:
                 bad = await session.call_tool(
                     "fetch", {"url": "https://example.com", "strategy": "bogus"}
                 )
-                assert not bad.isError or "error" in bad.content[0].text
+                assert not bad.is_error or "error" in bad.content[0].text
                 good = await session.call_tool(
                     "fetch", {"url": "https://example.com", "strategy": "http", "no_cache": True}
                 )

webget_cli-0.9.0/tests/test_mcp_profile.py

@@ -0,0 +1,177 @@
+"""Integration: MCP fetch with a real profile uses the stored session.
+
+A cookie-gated local server returns 200 only when the request carries the
+session cookie from a profile's storage_state.json:
+  - fetch(profile="testp")  -> success (session used)
+  - fetch() (anonymous)     -> blocked/login_required (no session)
+
+Spawns webget_mcp.py as a stdio subprocess with WEBGET_PROFILE_DIR pointing
+at a temp profile root and WEBGET_ALLOW_PRIVATE=1 (the test server lives on
+127.0.0.1, which the SSRF guard blocks by default).
+"""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+
+ROOT = Path(__file__).resolve().parent.parent
+MCP = ROOT / "webget_mcp.py"
+sys.path.insert(0, str(ROOT))
+
+from tests.http_server import TestServer
+
+
+def _spawn(profile_root):
+    env = {
+        **os.environ,
+        "WEBGET_PROFILE_DIR": str(profile_root),
+        "WEBGET_ALLOW_PRIVATE": "1",
+    }
+    return StdioServerParameters(command=sys.executable, args=[str(MCP)], env=env)
+
+
+def _make_profile(root, name):
+    d = root / name
+    (d / "Default").mkdir(parents=True)
+    (d / "storage_state.json").write_text(
+        json.dumps(
+            {
+                "cookies": [
+                    {
+                        "name": "session",
+                        "value": "1",
+                        "domain": "127.0.0.1",
+                        "path": "/",
+                        "expires": 9999999999,
+                    }
+                ]
+            }
+        )
+    )
+
+
+def test_mcp_fetch_with_profile_uses_session(tmp_path):
+    server = TestServer().start()
+    try:
+        root = tmp_path / "profiles"
+        _make_profile(root, "testp")
+        url = server.url("/cookie-gated")
+
+        async def run():
+            async with (
+                stdio_client(_spawn(root)) as (read, write),
+                ClientSession(read, write) as session,
+            ):
+                await session.initialize()
+                ok = await session.call_tool(
+                    "fetch", {"url": url, "strategy": "http", "no_cache": True, "profile": "testp"}
+                )
+                anon = await session.call_tool(
+                    "fetch", {"url": url, "strategy": "http", "no_cache": True}
+                )
+                return ok, anon
+
+        ok, anon = asyncio.run(asyncio.wait_for(run(), timeout=60))
+        ok_text = ok.content[0].text if ok.content else ""
+        anon_text = anon.content[0].text if anon.content else ""
+        assert not ok.is_error, ok_text
+        assert '"status":"success"' in ok_text, ok_text
+        assert "you are authenticated" in ok_text, ok_text
+        assert '"authenticated":true' in ok_text, ok_text  # session was used
+        # anonymous fetch must NOT succeed on the gated page
+        assert ok_text != anon_text
+        assert '"status":"success"' not in anon_text, f"anonymous fetch leaked: {anon_text}"
+    finally:
+        server.stop()
+
+
+def test_mcp_fetch_unknown_profile_is_hard_error(tmp_path):
+    """Even with a profile root, a name that does not exist is an error
+    (never a silent anonymous fallback)."""
+
+    async def run():
+        env = {**os.environ, "WEBGET_PROFILE_DIR": str(tmp_path / "profiles")}
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)], env=env)
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "fetch", {"url": "https://example.com", "profile": "ghost", "no_cache": True}
+            )
+            return res
+
+    res = asyncio.run(asyncio.wait_for(run(), timeout=60))
+    assert not res.is_error
+    assert "profile 'ghost' not found" in res.content[0].text
+
+
+def test_mcp_fetch_enriched_output_present(tmp_path):
+    """MCP fetch must include auth, attempts, and reasons in the response.
+
+    These fields give agents full provenance: whether the fetch used a
+    session, how many ladder steps were tried, and the full chain of
+    failures. Added 2026-08-31.
+    """
+    server = TestServer().start()
+    try:
+        root = tmp_path / "profiles"
+        _make_profile(root, "testp")
+        gated = server.url("/cookie-gated")
+
+        async def run():
+            async with (
+                stdio_client(_spawn(root)) as (read, write),
+                ClientSession(read, write) as session,
+            ):
+                await session.initialize()
+                # Authenticated fetch
+                ok = await session.call_tool(
+                    "fetch",
+                    {
+                        "url": gated,
+                        "strategy": "http",
+                        "no_cache": True,
+                        "profile": "testp",
+                    },
+                )
+                # Anonymous fetch (blocked)
+                anon = await session.call_tool(
+                    "fetch",
+                    {"url": gated, "strategy": "http", "no_cache": True},
+                )
+                return ok, anon
+
+        ok, anon = asyncio.run(asyncio.wait_for(run(), timeout=60))
+        ok_text = ok.content[0].text if ok.content else ""
+        anon_text = anon.content[0].text if anon.content else ""
+
+        # Authenticated: standard assertions
+        assert not ok.is_error, ok_text
+        assert '"status":"success"' in ok_text
+        # auth
+        assert '"auth"' in ok_text
+        assert '"authenticated":true' in ok_text
+        assert '"attempts"' in ok_text
+        # Success carries an empty reasons list (consistent shape: consumers
+        # can always iterate reasons without a None check).
+        assert '"reasons":[]' in ok_text, "success should have empty reasons"
+
+        # Anonymous (blocked): must have provenance fields
+        assert '"status":"blocked"' in anon_text or '"status":"login_required"' in anon_text
+        assert '"auth"' in anon_text
+        assert '"authenticated":false' in anon_text or '"authenticated":null' in anon_text
+        assert '"attempts"' in anon_text
+        assert '"reasons"' in anon_text
+        assert '"state":"blocked"' in anon_text or '"state":"login_required"' in anon_text
+    finally:
+        server.stop()
+
+
+if __name__ == "__main__":
+    import pytest
+
+    sys.exit(pytest.main([__file__, "-v"]))