webget-cli 0.8.0__tar.gz → 0.9.0__tar.gz

{webget_cli-0.8.0/webget_cli.egg-info → webget_cli-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.8.0
+Version: 0.9.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -247,7 +247,7 @@ standalone with `webget-mcp` (stdio transport) or `python webget_mcp.py`.
 ### Authenticated sessions (profiles)
 
 MCP tools can use locally stored login sessions. Create one first with the
-CLI:
+CLI, or let the agent create it via the MCP `login` tool:
 
 ```bash
 webget login https://portal.example.com --profile portal
@@ -257,13 +257,19 @@ Then the agent can discover sessions and fetch authenticated pages:
 
 - `list_profiles` - lists available sessions (name, last used, size,
   status). Cookie values are never returned.
+- `login(url, profile)` - opens a browser session (headful by default so a
+  human can log in), navigates to `url`, and persists the session once the
+  login handshake's cookies appear (or after `wait_seconds`, whichever
+  comes first). MCP stdin is the JSON-RPC stream, so there is no Enter
+  keypress; the flow polls for cookies instead.
 - `fetch(..., profile="portal")` / `search_fetch(..., profile="portal")` -
   scrape using that session.
 
 ```text
 agent: "check my portal for new announcements"
   1. list_profiles            -> portal (authenticated)
-  2. fetch(url, profile="portal")
+  2. login(https://portal.example.com, profile="portal")  # if not listed
+  3. fetch(url, profile="portal")
 ```
 
 Invalid profile names and unknown profiles are hard errors (no silent

{webget_cli-0.8.0 → webget_cli-0.9.0}/README.md

@@ -213,7 +213,7 @@ standalone with `webget-mcp` (stdio transport) or `python webget_mcp.py`.
 ### Authenticated sessions (profiles)
 
 MCP tools can use locally stored login sessions. Create one first with the
-CLI:
+CLI, or let the agent create it via the MCP `login` tool:
 
 ```bash
 webget login https://portal.example.com --profile portal
@@ -223,13 +223,19 @@ Then the agent can discover sessions and fetch authenticated pages:
 
 - `list_profiles` - lists available sessions (name, last used, size,
   status). Cookie values are never returned.
+- `login(url, profile)` - opens a browser session (headful by default so a
+  human can log in), navigates to `url`, and persists the session once the
+  login handshake's cookies appear (or after `wait_seconds`, whichever
+  comes first). MCP stdin is the JSON-RPC stream, so there is no Enter
+  keypress; the flow polls for cookies instead.
 - `fetch(..., profile="portal")` / `search_fetch(..., profile="portal")` -
   scrape using that session.
 
 ```text
 agent: "check my portal for new announcements"
   1. list_profiles            -> portal (authenticated)
-  2. fetch(url, profile="portal")
+  2. login(https://portal.example.com, profile="portal")  # if not listed
+  3. fetch(url, profile="portal")
 ```
 
 Invalid profile names and unknown profiles are hard errors (no silent

{webget_cli-0.8.0 → webget_cli-0.9.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.8.0"
+version = "0.9.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"

{webget_cli-0.8.0 → webget_cli-0.9.0}/tests/test_adversarial_http.py

@@ -1,6 +1,7 @@
 """Adversarial HTTP tests: URL handling, status codes, redirects, bodies."""
 
 import asyncio
+import time
 
 import pytest
 
@@ -63,6 +64,26 @@ class TestConnectionFailures:
         res = asyncio.run(_fetch(server.url("/timeout?sec=5"), per_url_timeout=1))
         assert _one(res)["status"] == "error"
 
+    def test_slow_drip_hits_wall_clock_deadline(self, fresh_cache):
+        """A server that slow-drips the body must not exceed per_url_timeout.
+
+        httpx's own timeout bounds a single socket operation only, so a
+        drip of small chunks every 150ms (total ~6s for 40 chunks) would
+        keep the request alive far past a 2s budget. fetch_http enforces
+        an absolute deadline while streaming; this proves the URL errors
+        out in roughly the budget, not the full drip duration.
+        """
+        server = fresh_cache
+        t0 = time.monotonic()
+        res = asyncio.run(
+            _fetch(server.url("/drip?n=40&delay=0.15"), per_url_timeout=2, strategy="http")
+        )
+        elapsed = time.monotonic() - t0
+        out = _one(res)
+        assert out["status"] == "error"
+        # Drip would take ~6s uncapped; deadline must cut it near 2s.
+        assert elapsed < 4.0, f"deadline not enforced: took {elapsed:.1f}s"
+
 
 # ---------- status codes ----------
 

{webget_cli-0.8.0 → webget_cli-0.9.0}/tests/test_adversarial_mcp.py

@@ -58,7 +58,7 @@ class TestMalformedArguments:
                 return res
 
         res = _run(run())
-        assert res.isError
+        assert res.is_error
 
 
 class TestInvalidURLs:
@@ -72,7 +72,7 @@ class TestInvalidURLs:
                 return res
 
         res = _run(run())
-        assert "error" in res.content[0].text or res.isError
+        assert "error" in res.content[0].text or res.is_error
 
 
 class TestRepeatedCalls:
@@ -86,7 +86,7 @@ class TestRepeatedCalls:
                         "fetch",
                         {"url": "https://example.com", "strategy": "http", "no_cache": True},
                     )
-                    if res.isError:
+                    if res.is_error:
                         return "ERROR"
                 return "OK"
 
@@ -106,7 +106,7 @@ class TestRepeatedCalls:
                         for _ in range(5)
                     ]
                 )
-                return [r.isError for r in results]
+                return [r.is_error for r in results]
 
         assert _run(run()) == [False] * 5
 
@@ -133,7 +133,7 @@ class TestInputCaps:
                 return res
 
         res = _run(run())
-        assert "must be between" in res.content[0].text or res.isError
+        assert "must be between" in res.content[0].text or res.is_error
 
 
 class TestToolFailureIsolation:

{webget_cli-0.8.0 → webget_cli-0.9.0}/tests/test_adversarial_ssrf.py

@@ -98,14 +98,14 @@ class TestSSRFBlocking:
         the initial URL passed the guard (e.g. it was public)."""
         import pytest
 
-        real = webget._is_private_target
+        real = webget._private_ip_for
         initial = server.url("/redirect-private")
 
         def guarded(url):
             # initial URL passes (as if public); every other URL checked for real
-            return real(url) if url != initial else False
+            return real(url) if url != initial else None
 
-        monkeypatch.setattr(webget, "_is_private_target", guarded)
+        monkeypatch.setattr(webget, "_private_ip_for", guarded)
         with pytest.raises(webget.SSRFError):
             asyncio.run(webget.fetch_http(initial, 2000, timeout=5))
 

webget_cli-0.9.0/tests/test_login_flow.py

@@ -0,0 +1,180 @@
+"""MCP login tool + non-interactive login flow tests.
+
+Covers:
+  - _login_flow(interactive=False): polls browser context until a Set-Cookie
+    arrives (here: /set-cookie on the local test server), then persists a
+    storage_state.json the HTTP fast path can reuse.
+  - MCP login tool: validates URL/profile, runs the flow headless against
+    the local server, then fetch(profile=...) uses the stored session.
+  - login tool rejects bad URLs and unknown/invalid profiles.
+
+The test server lives on 127.0.0.1, so WEBGET_ALLOW_PRIVATE=1 is required
+for the subsequent authenticated fetch through the SSRF-guarded HTTP path.
+"""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+import webget_cli as webget
+
+playwright = pytest.importorskip("playwright", reason="playwright not installed")
+
+
+def _profile_root(tmp_path):
+    return tmp_path / "profiles"
+
+
+def test_login_flow_noninteractive_persists_session(server, tmp_path, monkeypatch):
+    """Browser-less-enough path: headless Chromium navigates /set-cookie,
+    cookie polling sees the Set-Cookie side effect, storage_state persists,
+    and the HTTP fast path can reuse it on /cookie-gated."""
+    root = _profile_root(tmp_path)
+    monkeypatch.setattr(webget, "PROFILE_DIR", str(root))
+    monkeypatch.setenv("WEBGET_ALLOW_PRIVATE", "1")
+
+    asyncio.run(
+        webget._login_flow(
+            server.url("/set-cookie"),
+            "flowp",
+            headless=True,
+            wait_seconds=30,
+            interactive=False,
+            quiet=True,
+        )
+    )
+
+    state_p = webget.profile_state_path("flowp")
+    assert os.path.exists(state_p)
+    state = json.loads(Path(state_p).read_text())
+    cookies = {c["name"]: c["value"] for c in state["cookies"]}
+    assert cookies.get("webget_session") == "1"
+
+    # HTTP fast path must now authenticate on the cookie-gated page.
+    res = asyncio.run(
+        webget.scrape_many(
+            [server.url("/cookie-gated")],
+            strategy="http",
+            no_cache=True,
+            profile="flowp",
+        )
+    )
+    out = res[server.url("/cookie-gated")]
+    assert out["status"] == "success"
+    assert out["auth"]["authenticated"] is True
+    assert "you are authenticated" in out["markdown"]
+
+
+def test_login_flow_waits_until_cookie(server, tmp_path, monkeypatch):
+    """A page that does NOT set a cookie still persists (best-effort), and a
+    slow Set-Cookie is picked up by polling before the deadline."""
+    root = _profile_root(tmp_path)
+    monkeypatch.setattr(webget, "PROFILE_DIR", str(root))
+
+    asyncio.run(
+        webget._login_flow(
+            server.url("/set-cookie"),
+            "slowp",
+            headless=True,
+            wait_seconds=30,
+            interactive=False,
+            quiet=True,
+        )
+    )
+    state_p = webget.profile_state_path("slowp")
+    assert os.path.exists(state_p)
+    cookies = {c["name"] for c in json.loads(Path(state_p).read_text())["cookies"]}
+    assert "webget_session" in cookies
+
+
+def _spawn_mcp(profile_root):
+    from mcp import StdioServerParameters
+
+    env = {
+        **os.environ,
+        "WEBGET_PROFILE_DIR": str(profile_root),
+        "WEBGET_ALLOW_PRIVATE": "1",
+    }
+    return StdioServerParameters(command=sys.executable, args=[str(ROOT / "webget_mcp.py")], env=env)
+
+
+def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
+    """End-to-end MCP: login(url, profile, headless=True) against the local
+    /set-cookie page persists a session, then fetch(profile=...) on the
+    cookie-gated page succeeds while anonymous fetch does not."""
+    from mcp import ClientSession
+    from mcp.client.stdio import stdio_client
+
+    root = _profile_root(tmp_path)
+    login_url = server.url("/set-cookie")
+    gated_url = server.url("/cookie-gated")
+
+    async def run():
+        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "login",
+                {"url": login_url, "profile": "mcplogin", "headless": True, "wait_seconds": 30},
+            )
+            authed = await session.call_tool(
+                "fetch", {"url": gated_url, "strategy": "http", "no_cache": True, "profile": "mcplogin"}
+            )
+            anon = await session.call_tool(
+                "fetch", {"url": gated_url, "strategy": "http", "no_cache": True}
+            )
+            return res, authed, anon
+
+    res, authed, anon = asyncio.run(asyncio.wait_for(run(), timeout=90))
+    res_text = res.content[0].text if res.content else ""
+    assert not res.is_error, res_text
+    assert '"status":"success"' in res_text, res_text
+    assert '"profile":"mcplogin"' in res_text
+
+    authed_text = authed.content[0].text if authed.content else ""
+    assert '"status":"success"' in authed_text, authed_text
+    assert "you are authenticated" in authed_text
+    assert '"authenticated":true' in authed_text
+
+    anon_text = anon.content[0].text if anon.content else ""
+    assert '"status":"success"' not in anon_text, f"anonymous fetch leaked: {anon_text}"
+
+
+def test_mcp_login_rejects_bad_url_and_profile(server, tmp_path):
+    """login tool validates its inputs: bad URL and bad profile name are
+    clean errors, never a crash."""
+    from mcp import ClientSession
+    from mcp.client.stdio import stdio_client
+
+    root = _profile_root(tmp_path)
+
+    async def run():
+        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            bad_url = await session.call_tool(
+                "login", {"url": "not a url", "profile": "x", "headless": True, "wait_seconds": 5}
+            )
+            bad_name = await session.call_tool(
+                "login", {"url": server.url("/"), "profile": "../evil", "headless": True, "wait_seconds": 5}
+            )
+            bad_secs = await session.call_tool(
+                "login", {"url": server.url("/"), "profile": "x", "headless": True, "wait_seconds": 1}
+            )
+            return bad_url, bad_name, bad_secs
+
+    bad_url, bad_name, bad_secs = asyncio.run(asyncio.wait_for(run(), timeout=60))
+    assert "invalid site URL" in bad_url.content[0].text
+    assert "invalid profile name" in bad_name.content[0].text
+    assert "wait_seconds must be between 5 and 600" in bad_secs.content[0].text
+
+
+if __name__ == "__main__":
+    import pytest
+
+    sys.exit(pytest.main([__file__, "-v"]))

{webget_cli-0.8.0 → webget_cli-0.9.0}/tests/test_mcp_leak_review.py

@@ -64,7 +64,7 @@ class TestNoSecretLeakage:
                 return res
 
         res = _run(run())
-        if res.isError:
+        if res.is_error:
             return  # error path: no payload to leak, still fine
         text = res.content[0].text
         hits = _scan(text)
@@ -121,7 +121,7 @@ class TestServerRecovery:
                 bad = await session.call_tool(
                     "fetch", {"url": "https://example.com", "strategy": "bogus"}
                 )
-                assert not bad.isError or "error" in bad.content[0].text
+                assert not bad.is_error or "error" in bad.content[0].text
                 good = await session.call_tool(
                     "fetch", {"url": "https://example.com", "strategy": "http", "no_cache": True}
                 )

{webget_cli-0.8.0 → webget_cli-0.9.0}/tests/test_mcp_profile.py

@@ -43,7 +43,7 @@ def _make_profile(root, name):
             {
                 "cookies": [
                     {
-                        "name": "s",
+                        "name": "session",
                         "value": "1",
                         "domain": "127.0.0.1",
                         "path": "/",
@@ -79,7 +79,7 @@ def test_mcp_fetch_with_profile_uses_session(tmp_path):
         ok, anon = asyncio.run(asyncio.wait_for(run(), timeout=60))
         ok_text = ok.content[0].text if ok.content else ""
         anon_text = anon.content[0].text if anon.content else ""
-        assert not ok.isError, ok_text
+        assert not ok.is_error, ok_text
         assert '"status":"success"' in ok_text, ok_text
         assert "you are authenticated" in ok_text, ok_text
         assert '"authenticated":true' in ok_text, ok_text  # session was used
@@ -105,10 +105,72 @@ def test_mcp_fetch_unknown_profile_is_hard_error(tmp_path):
             return res
 
     res = asyncio.run(asyncio.wait_for(run(), timeout=60))
-    assert not res.isError
+    assert not res.is_error
     assert "profile 'ghost' not found" in res.content[0].text
 
 
+def test_mcp_fetch_enriched_output_present(tmp_path):
+    """MCP fetch must include auth, attempts, and reasons in the response.
+
+    These fields give agents full provenance: whether the fetch used a
+    session, how many ladder steps were tried, and the full chain of
+    failures. Added 2026-08-31.
+    """
+    server = TestServer().start()
+    try:
+        root = tmp_path / "profiles"
+        _make_profile(root, "testp")
+        gated = server.url("/cookie-gated")
+
+        async def run():
+            async with (
+                stdio_client(_spawn(root)) as (read, write),
+                ClientSession(read, write) as session,
+            ):
+                await session.initialize()
+                # Authenticated fetch
+                ok = await session.call_tool(
+                    "fetch",
+                    {
+                        "url": gated,
+                        "strategy": "http",
+                        "no_cache": True,
+                        "profile": "testp",
+                    },
+                )
+                # Anonymous fetch (blocked)
+                anon = await session.call_tool(
+                    "fetch",
+                    {"url": gated, "strategy": "http", "no_cache": True},
+                )
+                return ok, anon
+
+        ok, anon = asyncio.run(asyncio.wait_for(run(), timeout=60))
+        ok_text = ok.content[0].text if ok.content else ""
+        anon_text = anon.content[0].text if anon.content else ""
+
+        # Authenticated: standard assertions
+        assert not ok.is_error, ok_text
+        assert '"status":"success"' in ok_text
+        # auth
+        assert '"auth"' in ok_text
+        assert '"authenticated":true' in ok_text
+        assert '"attempts"' in ok_text
+        # Success carries an empty reasons list (consistent shape: consumers
+        # can always iterate reasons without a None check).
+        assert '"reasons":[]' in ok_text, "success should have empty reasons"
+
+        # Anonymous (blocked): must have provenance fields
+        assert '"status":"blocked"' in anon_text or '"status":"login_required"' in anon_text
+        assert '"auth"' in anon_text
+        assert '"authenticated":false' in anon_text or '"authenticated":null' in anon_text
+        assert '"attempts"' in anon_text
+        assert '"reasons"' in anon_text
+        assert '"state":"blocked"' in anon_text or '"state":"login_required"' in anon_text
+    finally:
+        server.stop()
+
+
 if __name__ == "__main__":
     import pytest
 

{webget_cli-0.8.0 → webget_cli-0.9.0}/tests/test_mcp_server.py

@@ -30,7 +30,9 @@ def test_tools_listed():
             return [t.name for t in tools.tools]
 
     # order is not a contract; membership is
-    assert sorted(_run(run())) == sorted(["search", "fetch", "search_fetch", "list_profiles"])
+    assert sorted(_run(run())) == sorted(
+        ["search", "fetch", "search_fetch", "list_profiles", "login"]
+    )
 
 
 def test_invalid_strategy_returns_error_not_crash():
@@ -51,7 +53,7 @@ def test_invalid_strategy_returns_error_not_crash():
             return res
 
     res = _run(run())
-    assert not res.isError
+    assert not res.is_error
     assert "error" in res.content[0].text
 
 
@@ -70,7 +72,7 @@ def test_firecrawl_without_key_returns_error_not_crash():
             return res
 
     res = _run(run())
-    assert not res.isError
+    assert not res.is_error
     assert "error" in res.content[0].text
 
 
@@ -93,7 +95,7 @@ def test_server_stays_alive_after_bad_calls():
             return res
 
     res = _run(run())
-    assert not res.isError
+    assert not res.is_error
     assert "success" in res.content[0].text
 
 
@@ -116,7 +118,7 @@ def test_fetch_invalid_profile_returns_error_not_crash():
             return res, ok
 
     res, ok = _run(run())
-    assert not res.isError
+    assert not res.is_error
     assert "invalid profile name" in res.content[0].text
     assert "success" in ok.content[0].text  # server alive after the bad call
 
@@ -137,7 +139,7 @@ def test_fetch_nonexistent_profile_returns_error():
             return res
 
     res = _run(run())
-    assert not res.isError
+    assert not res.is_error
     assert "profile 'ghost' not found" in res.content[0].text
 
 
@@ -159,9 +161,9 @@ def test_search_fetch_invalid_profile_returns_error_not_crash():
             return res, ok
 
     res, ok = _run(run())
-    assert not res.isError
+    assert not res.is_error
     assert "invalid profile name" in res.content[0].text
-    assert not ok.isError  # server alive after the bad call
+    assert not ok.is_error  # server alive after the bad call
 
 
 def test_list_profiles_tool_metadata_only(tmp_path):
@@ -187,7 +189,7 @@ def test_list_profiles_tool_metadata_only(tmp_path):
             return res
 
     res = _run(run())
-    assert not res.isError
+    assert not res.is_error
     text = res.content[0].text
     assert "sion" in text
     assert "SUPERSECRET" not in text  # cookie values never exposed

{webget_cli-0.8.0 → webget_cli-0.9.0}/tests/test_security_review.py

@@ -83,30 +83,30 @@ class TestRedirectHopPolicy:
         hop check must fire even though the initial URL passed."""
         import pytest
 
-        real = webget._is_private_target
+        real = webget._private_ip_for
         initial = server.url("/redirect-private")
 
         def guarded(url):
-            return real(url) if url != initial else False
+            return real(url) if url != initial else None
 
-        monkeypatch.setattr(webget, "_is_private_target", guarded)
+        monkeypatch.setattr(webget, "_private_ip_for", guarded)
         with pytest.raises(webget.SSRFError):
             asyncio.run(webget.fetch_http(initial, 2000, timeout=5))
 
     def test_multi_hop_into_private_blocked(self, server, isolated_env, monkeypatch):
         """public -> public -> private (3 hops) must still be blocked."""
 
-        real = webget._is_private_target
+        real = webget._private_ip_for
         initial = server.url("/redirect-chain?n=2")  # -> /redirect-chain?n=1 -> /normal
         # craft: first two hops pass (public), third hop lands on /private
         chain = [server.url("/redirect-chain?n=2"), server.url("/redirect-chain?n=1")]
 
         def guarded(url):
             if url in chain or url == server.url("/redirect-chain?n=2"):
-                return False
+                return None
             return real(url)
 
-        monkeypatch.setattr(webget, "_is_private_target", guarded)
+        monkeypatch.setattr(webget, "_private_ip_for", guarded)
         # fetch_http follows hops manually; every hop re-checks
         try:
             asyncio.run(webget.fetch_http(initial, 2000, timeout=5))