webget-cli 0.7.3__tar.gz → 0.8.0__tar.gz

{webget_cli-0.7.3 → webget_cli-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.7.3
+Version: 0.8.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -244,9 +244,31 @@ Register as a local MCP server in opencode:
 Then prompt with `use webget` for search and scrape tasks. Run the server
 standalone with `webget-mcp` (stdio transport) or `python webget_mcp.py`.
 
-> **Limitation:** MCP tools do not expose `--profile`/`--cookies`, so
-> authenticated pages are out of scope for the MCP server. Use the CLI
-> (`webget login`, `webget u --profile ...`) for session-based fetching.
+### Authenticated sessions (profiles)
+
+MCP tools can use locally stored login sessions. Create one first with the
+CLI:
+
+```bash
+webget login https://portal.example.com --profile portal
+```
+
+Then the agent can discover sessions and fetch authenticated pages:
+
+- `list_profiles` - lists available sessions (name, last used, size,
+  status). Cookie values are never returned.
+- `fetch(..., profile="portal")` / `search_fetch(..., profile="portal")` -
+  scrape using that session.
+
+```text
+agent: "check my portal for new announcements"
+  1. list_profiles            -> portal (authenticated)
+  2. fetch(url, profile="portal")
+```
+
+Invalid profile names and unknown profiles are hard errors (no silent
+anonymous fallback). Sessions are local to the machine running the MCP
+server.
 
 ## Development
 

{webget_cli-0.7.3 → webget_cli-0.8.0}/README.md

@@ -210,9 +210,31 @@ Register as a local MCP server in opencode:
 Then prompt with `use webget` for search and scrape tasks. Run the server
 standalone with `webget-mcp` (stdio transport) or `python webget_mcp.py`.
 
-> **Limitation:** MCP tools do not expose `--profile`/`--cookies`, so
-> authenticated pages are out of scope for the MCP server. Use the CLI
-> (`webget login`, `webget u --profile ...`) for session-based fetching.
+### Authenticated sessions (profiles)
+
+MCP tools can use locally stored login sessions. Create one first with the
+CLI:
+
+```bash
+webget login https://portal.example.com --profile portal
+```
+
+Then the agent can discover sessions and fetch authenticated pages:
+
+- `list_profiles` - lists available sessions (name, last used, size,
+  status). Cookie values are never returned.
+- `fetch(..., profile="portal")` / `search_fetch(..., profile="portal")` -
+  scrape using that session.
+
+```text
+agent: "check my portal for new announcements"
+  1. list_profiles            -> portal (authenticated)
+  2. fetch(url, profile="portal")
+```
+
+Invalid profile names and unknown profiles are hard errors (no silent
+anonymous fallback). Sessions are local to the machine running the MCP
+server.
 
 ## Development
 

{webget_cli-0.7.3 → webget_cli-0.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.7.3"
+version = "0.8.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"

webget_cli-0.8.0/tests/test_mcp_profile.py

@@ -0,0 +1,115 @@
+"""Integration: MCP fetch with a real profile uses the stored session.
+
+A cookie-gated local server returns 200 only when the request carries the
+session cookie from a profile's storage_state.json:
+  - fetch(profile="testp")  -> success (session used)
+  - fetch() (anonymous)     -> blocked/login_required (no session)
+
+Spawns webget_mcp.py as a stdio subprocess with WEBGET_PROFILE_DIR pointing
+at a temp profile root and WEBGET_ALLOW_PRIVATE=1 (the test server lives on
+127.0.0.1, which the SSRF guard blocks by default).
+"""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+
+ROOT = Path(__file__).resolve().parent.parent
+MCP = ROOT / "webget_mcp.py"
+sys.path.insert(0, str(ROOT))
+
+from tests.http_server import TestServer
+
+
+def _spawn(profile_root):
+    env = {
+        **os.environ,
+        "WEBGET_PROFILE_DIR": str(profile_root),
+        "WEBGET_ALLOW_PRIVATE": "1",
+    }
+    return StdioServerParameters(command=sys.executable, args=[str(MCP)], env=env)
+
+
+def _make_profile(root, name):
+    d = root / name
+    (d / "Default").mkdir(parents=True)
+    (d / "storage_state.json").write_text(
+        json.dumps(
+            {
+                "cookies": [
+                    {
+                        "name": "s",
+                        "value": "1",
+                        "domain": "127.0.0.1",
+                        "path": "/",
+                        "expires": 9999999999,
+                    }
+                ]
+            }
+        )
+    )
+
+
+def test_mcp_fetch_with_profile_uses_session(tmp_path):
+    server = TestServer().start()
+    try:
+        root = tmp_path / "profiles"
+        _make_profile(root, "testp")
+        url = server.url("/cookie-gated")
+
+        async def run():
+            async with (
+                stdio_client(_spawn(root)) as (read, write),
+                ClientSession(read, write) as session,
+            ):
+                await session.initialize()
+                ok = await session.call_tool(
+                    "fetch", {"url": url, "strategy": "http", "no_cache": True, "profile": "testp"}
+                )
+                anon = await session.call_tool(
+                    "fetch", {"url": url, "strategy": "http", "no_cache": True}
+                )
+                return ok, anon
+
+        ok, anon = asyncio.run(asyncio.wait_for(run(), timeout=60))
+        ok_text = ok.content[0].text if ok.content else ""
+        anon_text = anon.content[0].text if anon.content else ""
+        assert not ok.isError, ok_text
+        assert '"status":"success"' in ok_text, ok_text
+        assert "you are authenticated" in ok_text, ok_text
+        assert '"authenticated":true' in ok_text, ok_text  # session was used
+        # anonymous fetch must NOT succeed on the gated page
+        assert ok_text != anon_text
+        assert '"status":"success"' not in anon_text, f"anonymous fetch leaked: {anon_text}"
+    finally:
+        server.stop()
+
+
+def test_mcp_fetch_unknown_profile_is_hard_error(tmp_path):
+    """Even with a profile root, a name that does not exist is an error
+    (never a silent anonymous fallback)."""
+
+    async def run():
+        env = {**os.environ, "WEBGET_PROFILE_DIR": str(tmp_path / "profiles")}
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)], env=env)
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "fetch", {"url": "https://example.com", "profile": "ghost", "no_cache": True}
+            )
+            return res
+
+    res = asyncio.run(asyncio.wait_for(run(), timeout=60))
+    assert not res.isError
+    assert "profile 'ghost' not found" in res.content[0].text
+
+
+if __name__ == "__main__":
+    import pytest
+
+    sys.exit(pytest.main([__file__, "-v"]))

webget_cli-0.8.0/tests/test_mcp_server.py

@@ -0,0 +1,199 @@
+"""MCP server regression tests.
+
+Spawns webget_mcp.py as a stdio subprocess and drives it like an MCP
+client would. Requires the mcp client SDK (installed via the [mcp] extra).
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+
+ROOT = Path(__file__).resolve().parent.parent
+MCP = ROOT / "webget_mcp.py"
+
+
+def _run(coro, timeout=60):
+    """Run a coroutine with a hard wall-clock cap so a dead server
+    (the bug this guards against) fails the test instead of hanging CI."""
+    return asyncio.run(asyncio.wait_for(coro, timeout=timeout))
+
+
+def test_tools_listed():
+    async def run():
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            tools = await session.list_tools()
+            return [t.name for t in tools.tools]
+
+    # order is not a contract; membership is
+    assert sorted(_run(run())) == sorted(["search", "fetch", "search_fetch", "list_profiles"])
+
+
+def test_invalid_strategy_returns_error_not_crash():
+    """The server must survive a client sending strategy='bogus'.
+
+    Before the guard, _ladder raised SystemExit (a BaseException) which
+    killed the whole server process and hung the client.
+    """
+
+    async def run():
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "fetch",
+                {"url": "https://example.com", "strategy": "bogus", "no_cache": True},
+            )
+            return res
+
+    res = _run(run())
+    assert not res.isError
+    assert "error" in res.content[0].text
+
+
+def test_firecrawl_without_key_returns_error_not_crash():
+    """strategy='firecrawl' without WEBGET_FIRECRAWL_KEY must not kill the
+    server either (SystemExit from _ladder)."""
+
+    async def run():
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "fetch",
+                {"url": "https://example.com", "strategy": "firecrawl", "no_cache": True},
+            )
+            return res
+
+    res = _run(run())
+    assert not res.isError
+    assert "error" in res.content[0].text
+
+
+def test_server_stays_alive_after_bad_calls():
+    """The same server process that handled a bad strategy call must still
+    answer a normal fetch afterwards."""
+
+    async def run():
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            await session.call_tool(
+                "fetch",
+                {"url": "https://example.com", "strategy": "bogus", "no_cache": True},
+            )
+            res = await session.call_tool(
+                "fetch",
+                {"url": "https://example.com", "strategy": "http", "no_cache": True},
+            )
+            return res
+
+    res = _run(run())
+    assert not res.isError
+    assert "success" in res.content[0].text
+
+
+def test_fetch_invalid_profile_returns_error_not_crash():
+    """profile name with path traversal must return a clean error and the
+    server must stay alive (SystemExit from profile_dir must not escape)."""
+
+    async def run():
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "fetch",
+                {"url": "https://example.com", "profile": "../etc", "no_cache": True},
+            )
+            ok = await session.call_tool(
+                "fetch",
+                {"url": "https://example.com", "strategy": "http", "no_cache": True},
+            )
+            return res, ok
+
+    res, ok = _run(run())
+    assert not res.isError
+    assert "invalid profile name" in res.content[0].text
+    assert "success" in ok.content[0].text  # server alive after the bad call
+
+
+def test_fetch_nonexistent_profile_returns_error():
+    """A valid-looking profile that does not exist is a hard error, never a
+    silent anonymous fallback (which would return login_required while the
+    agent believes the session was used)."""
+
+    async def run():
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "fetch",
+                {"url": "https://example.com", "profile": "ghost", "no_cache": True},
+            )
+            return res
+
+    res = _run(run())
+    assert not res.isError
+    assert "profile 'ghost' not found" in res.content[0].text
+
+
+def test_search_fetch_invalid_profile_returns_error_not_crash():
+    """search_fetch must validate profile the same way and stay alive."""
+
+    async def run():
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool(
+                "search_fetch",
+                {"query": "test", "n": 1, "profile": "../etc"},
+            )
+            ok = await session.call_tool(
+                "search_fetch",
+                {"query": "example.com", "n": 1},
+            )
+            return res, ok
+
+    res, ok = _run(run())
+    assert not res.isError
+    assert "invalid profile name" in res.content[0].text
+    assert not ok.isError  # server alive after the bad call
+
+
+def test_list_profiles_tool_metadata_only(tmp_path):
+    """list_profiles must list the session and NEVER leak cookie values."""
+    import json
+    import os
+
+    root = tmp_path / "profiles"
+    d = root / "sion"
+    (d / "Default").mkdir(parents=True)
+    (d / "storage_state.json").write_text(
+        json.dumps(
+            {"cookies": [{"name": "s", "value": "SUPERSECRET", "domain": ".x.com", "expires": -1}]}
+        )
+    )
+
+    async def run():
+        env = {**os.environ, "WEBGET_PROFILE_DIR": str(root)}
+        params = StdioServerParameters(command=sys.executable, args=[str(MCP)], env=env)
+        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+            res = await session.call_tool("list_profiles", {})
+            return res
+
+    res = _run(run())
+    assert not res.isError
+    text = res.content[0].text
+    assert "sion" in text
+    assert "SUPERSECRET" not in text  # cookie values never exposed
+
+
+if __name__ == "__main__":
+    import pytest
+
+    sys.exit(pytest.main([__file__, "-v"]))

{webget_cli-0.7.3 → webget_cli-0.8.0}/tests/test_webget.py

@@ -85,6 +85,47 @@ class TestProfileSafety:
         p = webget.profile_state_path("campus")
         assert p.endswith(os.path.join("campus", "storage_state.json"))
 
+    def test_env_override(self, monkeypatch):
+        monkeypatch.setenv("WEBGET_PROFILE_DIR", "/tmp/override-profiles")
+        assert webget.profile_dir("campus").startswith("/tmp/override-profiles")
+        assert webget.profile_state_path("campus").startswith("/tmp/override-profiles")
+
+
+# ---------- list_profiles / profile_exists (MCP reuse) ----------
+
+
+class TestListProfiles:
+    def test_empty_root(self, tmp_path, monkeypatch):
+        monkeypatch.setattr(webget, "PROFILE_DIR", str(tmp_path / "profiles"))
+        assert webget.list_profiles() == []
+
+    def test_returns_metadata_only(self, tmp_path, monkeypatch):
+        import json
+
+        monkeypatch.setattr(webget, "PROFILE_DIR", str(tmp_path / "profiles"))
+        d = tmp_path / "profiles" / "sion"
+        (d / "Default").mkdir(parents=True)
+        state = {
+            "cookies": [
+                {"name": "secret", "value": "SUPERSECRET", "domain": ".x.com", "expires": -1}
+            ]
+        }
+        (d / "storage_state.json").write_text(json.dumps(state))
+        meta = webget.list_profiles()
+        assert len(meta) == 1
+        assert meta[0]["profile"] == "sion"
+        assert "status" in meta[0]
+        # cookie values must NEVER appear in the listing
+        assert "SUPERSECRET" not in json.dumps(meta)
+
+    def test_profile_exists(self, tmp_path, monkeypatch):
+        monkeypatch.setattr(webget, "PROFILE_DIR", str(tmp_path / "profiles"))
+        (tmp_path / "profiles" / "sion").mkdir(parents=True)
+        assert webget.profile_exists("sion") is True
+        assert webget.profile_exists("nope") is False
+        assert webget.profile_exists("../evil") is False  # invalid = not exists
+        assert webget.profile_exists("") is False
+
 
 # ---------- auth state classifier ----------
 

{webget_cli-0.7.3 → webget_cli-0.8.0}/webget_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.7.3
+Version: 0.8.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -244,9 +244,31 @@ Register as a local MCP server in opencode:
 Then prompt with `use webget` for search and scrape tasks. Run the server
 standalone with `webget-mcp` (stdio transport) or `python webget_mcp.py`.
 
-> **Limitation:** MCP tools do not expose `--profile`/`--cookies`, so
-> authenticated pages are out of scope for the MCP server. Use the CLI
-> (`webget login`, `webget u --profile ...`) for session-based fetching.
+### Authenticated sessions (profiles)
+
+MCP tools can use locally stored login sessions. Create one first with the
+CLI:
+
+```bash
+webget login https://portal.example.com --profile portal
+```
+
+Then the agent can discover sessions and fetch authenticated pages:
+
+- `list_profiles` - lists available sessions (name, last used, size,
+  status). Cookie values are never returned.
+- `fetch(..., profile="portal")` / `search_fetch(..., profile="portal")` -
+  scrape using that session.
+
+```text
+agent: "check my portal for new announcements"
+  1. list_profiles            -> portal (authenticated)
+  2. fetch(url, profile="portal")
+```
+
+Invalid profile names and unknown profiles are hard errors (no silent
+anonymous fallback). Sessions are local to the machine running the MCP
+server.
 
 ## Development
 

{webget_cli-0.7.3 → webget_cli-0.8.0}/webget_cli.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ tests/test_extraction_markdown.py
 tests/test_firecrawl_policy.py
 tests/test_integration_ladder.py
 tests/test_mcp_leak_review.py
+tests/test_mcp_profile.py
 tests/test_mcp_server.py
 tests/test_mcp_smoke.py
 tests/test_security_review.py

{webget_cli-0.7.3 → webget_cli-0.8.0}/webget_cli.py

@@ -517,7 +517,22 @@ async def _guard_browser_routes(crawler_ctx):
     browser.on("context", lambda ctx: asyncio.create_task(register(ctx)))
 
 
-PROFILE_DIR = os.path.expanduser("~/.local/share/webget/profiles")
+def _profile_root():
+    """Profile root dir; WEBGET_PROFILE_DIR env override for tests/ops.
+
+    Order: env override first, then a monkeypatched/overridden PROFILE_DIR
+    module attribute (existing tests patch it directly), then the default.
+    """
+    env = os.environ.get("WEBGET_PROFILE_DIR")
+    if env:
+        return env
+    patched = globals().get("PROFILE_DIR")
+    if patched:
+        return patched
+    return os.path.expanduser("~/.local/share/webget/profiles")
+
+
+PROFILE_DIR = _profile_root()
 _PROFILE_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+$")
 
 
@@ -525,7 +540,7 @@ def profile_dir(name):
     """Return profile dir, rejecting path traversal / weird names."""
     if not name or not _PROFILE_NAME_RE.match(name) or name in (".", ".."):
         raise SystemExit(f"invalid profile name: {name!r}")
-    return os.path.join(PROFILE_DIR, name)
+    return os.path.join(_profile_root(), name)
 
 
 def profile_state_path(profile):
@@ -1046,25 +1061,41 @@ def _fmt_age(ts):
     return f"{int(diff // 86400)}d ago"
 
 
-def cmd_profiles(json_out):
-    if not os.path.isdir(PROFILE_DIR):
-        if json_out:
-            print("{}")
-        else:
-            print("PROFILE     LAST USED     SIZE       STATUS\n(no profiles yet)")
-        return
-    profiles = []
-    for name in sorted(os.listdir(PROFILE_DIR)):
+def list_profiles():
+    """Non-sensitive metadata for every profile dir. Never cookie values."""
+    if not os.path.isdir(_profile_root()):
+        return []
+    out = []
+    for name in sorted(os.listdir(_profile_root())):
         if not os.path.isdir(profile_dir(name)):
             continue
         try:
-            profile_dir(name)  # validates name; raises SystemExit on bad names
+            profile_dir(name)  # validates; raises SystemExit on bad names
             meta = _profile_meta(name)
             if meta["size"] > 0 or meta["status"] != "unknown":
-                profiles.append(meta)
+                out.append(meta)
         except SystemExit:
             # Skip malformed dir names that fail validation (e.g. "..").
             continue
+    return out
+
+
+def profile_exists(name):
+    """True if name is a valid profile name AND its dir exists."""
+    try:
+        return os.path.isdir(profile_dir(name))
+    except SystemExit:
+        return False
+
+
+def cmd_profiles(json_out):
+    if not os.path.isdir(_profile_root()):
+        if json_out:
+            print("{}")
+        else:
+            print("PROFILE     LAST USED     SIZE       STATUS\n(no profiles yet)")
+        return
+    profiles = list_profiles()
     if json_out:
         print(json.dumps({p["profile"]: p for p in profiles}, indent=2))
         return

{webget_cli-0.7.3 → webget_cli-0.8.0}/webget_mcp.py

@@ -22,6 +22,7 @@ Dependencies: mcp (pip install mcp). webget_cli.py must live next to this file.
 from __future__ import annotations
 
 import asyncio
+import os
 import sys
 from pathlib import Path
 
@@ -52,6 +53,23 @@ def _clamp(name, value, lo, hi):
     return None
 
 
+def _validate_profile(profile):
+    """Validate a profile name for authenticated fetches.
+
+    Returns an error string, or None when the profile exists and is usable.
+    Invalid names (path traversal etc.) and unknown profiles are HARD errors:
+    a silent anonymous fallback would return login_required while the agent
+    believes its session was used.
+    """
+    try:
+        pdir = wg.profile_dir(profile)  # raises SystemExit on invalid names
+    except SystemExit:
+        return f"invalid profile name: {profile!r}"
+    if not os.path.isdir(pdir):
+        return f"profile '{profile}' not found"
+    return None
+
+
 @mcp.tool()
 async def search(query: str, n: int = 5) -> list[dict]:
     """Search the web (DuckDuckGo). Returns up to n results with title/url/snippet."""
@@ -61,6 +79,14 @@ async def search(query: str, n: int = 5) -> list[dict]:
     return await asyncio.to_thread(wg.search, query, n)
 
 
+@mcp.tool()
+def list_profiles() -> list[dict]:
+    """List locally stored login sessions (profiles) with non-sensitive
+    metadata: name, last used, size, status. Cookie values are never
+    returned. Use with fetch(profile=...) to scrape authenticated pages."""
+    return wg.list_profiles()
+
+
 @mcp.tool()
 async def fetch(
     url: str,
@@ -68,11 +94,16 @@ async def fetch(
     timeout: int = 20,
     strategy: str = "auto",
     no_cache: bool = False,
+    profile: str | None = None,
 ) -> dict:
     """Scrape a single URL to markdown, with metadata.
 
     strategy: auto|http|crawl4ai|firecrawl. Status values:
     success|login_required|challenge|blocked|error.
+
+    profile: name of a locally stored login session (created with
+    'webget login URL --profile NAME'). Invalid names and unknown
+    profiles are hard errors (never a silent anonymous fallback).
     """
     if strategy not in _VALID_STRATEGIES:
         return {"status": "error", "error": f"unknown strategy: {strategy}"}
@@ -85,12 +116,17 @@ async def fetch(
         err = _clamp(name, value, lo, hi)
         if err:
             return {"status": "error", "error": err}
+    if profile is not None:
+        err = _validate_profile(profile)
+        if err:
+            return {"status": "error", "error": err}
     res = await wg.scrape_many(
         [url],
         max_chars=max_chars,
         per_url_timeout=timeout,
         strategy=strategy,
         no_cache=no_cache,
+        profile=profile,
     )
     return res.get(url, {})
 
@@ -102,11 +138,16 @@ async def search_fetch(
     max_chars: int = 4000,
     timeout: int = 20,
     no_cache: bool = False,
+    profile: str | None = None,
 ) -> list[dict]:
     """Search the web, then scrape the top n results in parallel.
 
     Returns one entry per URL with rank, search snippet, and scrape result
     (status/method/markdown).
+
+    profile: name of a locally stored login session (created with
+    'webget login URL --profile NAME'). Invalid names and unknown
+    profiles are hard errors (never a silent anonymous fallback).
     """
     for name, value, lo, hi in (
         ("n", n, 1, _MAX_SEARCH_N),
@@ -116,6 +157,10 @@ async def search_fetch(
         err = _clamp(name, value, lo, hi)
         if err:
             return [{"error": err}]
+    if profile is not None:
+        err = _validate_profile(profile)
+        if err:
+            return [{"error": err}]
     results = await asyncio.to_thread(wg.search, query, n)
     urls = [r["url"] for r in results]
     scraped = await wg.scrape_many(
@@ -123,6 +168,7 @@ async def search_fetch(
         max_chars=max_chars,
         per_url_timeout=timeout,
         no_cache=no_cache,
+        profile=profile,
     )
     out = []
     for i, r in enumerate(results):

webget_cli-0.7.3/tests/test_mcp_server.py

@@ -1,102 +0,0 @@
-"""MCP server regression tests.
-
-Spawns webget_mcp.py as a stdio subprocess and drives it like an MCP
-client would. Requires the mcp client SDK (installed via the [mcp] extra).
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-from mcp import ClientSession, StdioServerParameters
-from mcp.client.stdio import stdio_client
-
-ROOT = Path(__file__).resolve().parent.parent
-MCP = ROOT / "webget_mcp.py"
-
-
-def _run(coro, timeout=60):
-    """Run a coroutine with a hard wall-clock cap so a dead server
-    (the bug this guards against) fails the test instead of hanging CI."""
-    return asyncio.run(asyncio.wait_for(coro, timeout=timeout))
-
-
-def test_tools_listed():
-    async def run():
-        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
-        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
-            await session.initialize()
-            tools = await session.list_tools()
-            return [t.name for t in tools.tools]
-
-    assert _run(run()) == ["search", "fetch", "search_fetch"]
-
-
-def test_invalid_strategy_returns_error_not_crash():
-    """The server must survive a client sending strategy='bogus'.
-
-    Before the guard, _ladder raised SystemExit (a BaseException) which
-    killed the whole server process and hung the client.
-    """
-
-    async def run():
-        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
-        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
-            await session.initialize()
-            res = await session.call_tool(
-                "fetch",
-                {"url": "https://example.com", "strategy": "bogus", "no_cache": True},
-            )
-            return res
-
-    res = _run(run())
-    assert not res.isError
-    assert "error" in res.content[0].text
-
-
-def test_firecrawl_without_key_returns_error_not_crash():
-    """strategy='firecrawl' without WEBGET_FIRECRAWL_KEY must not kill the
-    server either (SystemExit from _ladder)."""
-
-    async def run():
-        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
-        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
-            await session.initialize()
-            res = await session.call_tool(
-                "fetch",
-                {"url": "https://example.com", "strategy": "firecrawl", "no_cache": True},
-            )
-            return res
-
-    res = _run(run())
-    assert not res.isError
-    assert "error" in res.content[0].text
-
-
-def test_server_stays_alive_after_bad_calls():
-    """The same server process that handled a bad strategy call must still
-    answer a normal fetch afterwards."""
-
-    async def run():
-        params = StdioServerParameters(command=sys.executable, args=[str(MCP)])
-        async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
-            await session.initialize()
-            await session.call_tool(
-                "fetch",
-                {"url": "https://example.com", "strategy": "bogus", "no_cache": True},
-            )
-            res = await session.call_tool(
-                "fetch",
-                {"url": "https://example.com", "strategy": "http", "no_cache": True},
-            )
-            return res
-
-    res = _run(run())
-    assert not res.isError
-    assert "success" in res.content[0].text
-
-
-if __name__ == "__main__":
-    import pytest
-
-    sys.exit(pytest.main([__file__, "-v"]))