webget-cli 0.9.0__tar.gz → 0.11.0__tar.gz

{webget_cli-0.9.0 → webget_cli-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.9.0
+Version: 0.11.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0

{webget_cli-0.9.0 → webget_cli-0.11.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.9.0"
+version = "0.11.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -39,11 +39,12 @@ mcp = ["fastmcp>=2"]
 dev = ["pytest>=8", "ruff>=0.6"]
 
 [project.scripts]
-webget = "webget_cli:main"
+webget = "webget.cli:main"
 webget-mcp = "webget_mcp:main"
 
 [tool.setuptools]
 py-modules = ["webget_cli", "webget_mcp"]
+packages = ["webget"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

{webget_cli-0.9.0 → webget_cli-0.11.0}/tests/test_adversarial_mcp.py

@@ -58,7 +58,7 @@ class TestMalformedArguments:
                 return res
 
         res = _run(run())
-        assert res.is_error
+        assert res.isError
 
 
 class TestInvalidURLs:
@@ -72,7 +72,7 @@ class TestInvalidURLs:
                 return res
 
         res = _run(run())
-        assert "error" in res.content[0].text or res.is_error
+        assert "error" in res.content[0].text or res.isError
 
 
 class TestRepeatedCalls:
@@ -86,7 +86,7 @@ class TestRepeatedCalls:
                         "fetch",
                         {"url": "https://example.com", "strategy": "http", "no_cache": True},
                     )
-                    if res.is_error:
+                    if res.isError:
                         return "ERROR"
                 return "OK"
 
@@ -106,7 +106,7 @@ class TestRepeatedCalls:
                         for _ in range(5)
                     ]
                 )
-                return [r.is_error for r in results]
+                return [r.isError for r in results]
 
         assert _run(run()) == [False] * 5
 
@@ -133,7 +133,7 @@ class TestInputCaps:
                 return res
 
         res = _run(run())
-        assert "must be between" in res.content[0].text or res.is_error
+        assert "must be between" in res.content[0].text or res.isError
 
 
 class TestToolFailureIsolation:

webget_cli-0.11.0/tests/test_discovery_map.py

@@ -0,0 +1,29 @@
+import asyncio
+
+from webget.discovery import _extract_sitemap_urls, discover_urls
+
+
+def test_extract_sitemap_urls_standard_xml():
+    xml = """<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+            <loc>https://example.com/page1</loc>
+        </url>
+        <url>
+            <loc>https://example.com/page2</loc>
+        </url>
+    </urlset>
+    """
+    urls = _extract_sitemap_urls(xml)
+    assert urls == ["https://example.com/page1", "https://example.com/page2"]
+
+
+def test_extract_sitemap_urls_malformed_regex_fallback():
+    xml = """<urlset><url><loc>https://example.com/broken1</loc></unclosed>"""
+    urls = _extract_sitemap_urls(xml)
+    assert "https://example.com/broken1" in urls
+
+
+def test_discover_urls_private_target_blocked():
+    urls = asyncio.run(discover_urls("http://127.0.0.1/sitemap.xml", allow_private=False))
+    assert urls == []

webget_cli-0.11.0/tests/test_ladder_retry.py

@@ -0,0 +1,41 @@
+import asyncio
+from unittest.mock import patch
+
+from webget.ladder import scrape_many
+
+
+def test_scrape_many_retry_transient_timeout(fresh_cache, server):
+    calls = []
+
+    async def mock_fetch(url, *args, **kwargs):
+        calls.append(url)
+        if len(calls) == 1:
+            raise TimeoutError("timeout")
+        return {
+            "title": "Success After Retry",
+            "markdown": "Valid content length " * 10,
+            "status": "success",
+        }
+
+    async def run_without_retry():
+        return await scrape_many(
+            [server.url("/test")], strategy="http", per_url_timeout=1, retry_transient=False
+        )
+
+    async def run_with_retry():
+        return await scrape_many(
+            [server.url("/test")], strategy="http", per_url_timeout=1, retry_transient=True
+        )
+
+    with patch("webget.ladder._resolve_fetch_http", return_value=mock_fetch):
+        # Without retry flag -> error on first timeout
+        res = asyncio.run(run_without_retry())
+        target = server.url("/test")
+        assert res[target]["status"] == "error"
+        assert res[target]["attempts"] == 1
+
+        # With retry_transient=True -> retries and succeeds on 2nd attempt
+        calls.clear()
+        res = asyncio.run(run_with_retry())
+        assert res[target]["status"] == "success"
+        assert res[target]["attempts"] == 2

{webget_cli-0.9.0 → webget_cli-0.11.0}/tests/test_login_flow.py

@@ -102,7 +102,9 @@ def _spawn_mcp(profile_root):
         "WEBGET_PROFILE_DIR": str(profile_root),
         "WEBGET_ALLOW_PRIVATE": "1",
     }
-    return StdioServerParameters(command=sys.executable, args=[str(ROOT / "webget_mcp.py")], env=env)
+    return StdioServerParameters(
+        command=sys.executable, args=[str(ROOT / "webget_mcp.py")], env=env
+    )
 
 
 def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
@@ -117,14 +119,18 @@ def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
     gated_url = server.url("/cookie-gated")
 
     async def run():
-        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+        async with (
+            stdio_client(_spawn_mcp(root)) as (read, write),
+            ClientSession(read, write) as session,
+        ):
             await session.initialize()
             res = await session.call_tool(
                 "login",
                 {"url": login_url, "profile": "mcplogin", "headless": True, "wait_seconds": 30},
             )
             authed = await session.call_tool(
-                "fetch", {"url": gated_url, "strategy": "http", "no_cache": True, "profile": "mcplogin"}
+                "fetch",
+                {"url": gated_url, "strategy": "http", "no_cache": True, "profile": "mcplogin"},
             )
             anon = await session.call_tool(
                 "fetch", {"url": gated_url, "strategy": "http", "no_cache": True}
@@ -133,7 +139,7 @@ def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
 
     res, authed, anon = asyncio.run(asyncio.wait_for(run(), timeout=90))
     res_text = res.content[0].text if res.content else ""
-    assert not res.is_error, res_text
+    assert not res.isError, res_text
     assert '"status":"success"' in res_text, res_text
     assert '"profile":"mcplogin"' in res_text
 
@@ -155,16 +161,21 @@ def test_mcp_login_rejects_bad_url_and_profile(server, tmp_path):
     root = _profile_root(tmp_path)
 
     async def run():
-        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+        async with (
+            stdio_client(_spawn_mcp(root)) as (read, write),
+            ClientSession(read, write) as session,
+        ):
             await session.initialize()
             bad_url = await session.call_tool(
                 "login", {"url": "not a url", "profile": "x", "headless": True, "wait_seconds": 5}
             )
             bad_name = await session.call_tool(
-                "login", {"url": server.url("/"), "profile": "../evil", "headless": True, "wait_seconds": 5}
+                "login",
+                {"url": server.url("/"), "profile": "../evil", "headless": True, "wait_seconds": 5},
             )
             bad_secs = await session.call_tool(
-                "login", {"url": server.url("/"), "profile": "x", "headless": True, "wait_seconds": 1}
+                "login",
+                {"url": server.url("/"), "profile": "x", "headless": True, "wait_seconds": 1},
             )
             return bad_url, bad_name, bad_secs
 

{webget_cli-0.9.0 → webget_cli-0.11.0}/tests/test_mcp_leak_review.py

@@ -64,7 +64,7 @@ class TestNoSecretLeakage:
                 return res
 
         res = _run(run())
-        if res.is_error:
+        if res.isError:
             return  # error path: no payload to leak, still fine
         text = res.content[0].text
         hits = _scan(text)
@@ -121,7 +121,7 @@ class TestServerRecovery:
                 bad = await session.call_tool(
                     "fetch", {"url": "https://example.com", "strategy": "bogus"}
                 )
-                assert not bad.is_error or "error" in bad.content[0].text
+                assert not bad.isError or "error" in bad.content[0].text
                 good = await session.call_tool(
                     "fetch", {"url": "https://example.com", "strategy": "http", "no_cache": True}
                 )

webget_cli-0.11.0/tests/test_mcp_map.py

@@ -0,0 +1,18 @@
+import asyncio
+from unittest.mock import AsyncMock, patch
+
+from webget_mcp import map as mcp_map
+
+
+def test_mcp_map_clamps_limit():
+    res = asyncio.run(mcp_map("https://example.com", limit=-5))
+    assert len(res) == 1
+    assert "error: limit must be between" in res[0]
+
+
+def test_mcp_map_calls_discover_urls():
+    with patch("webget_cli.discover_urls", new_callable=AsyncMock) as mock_disc:
+        mock_disc.return_value = ["https://example.com/p1", "https://example.com/p2"]
+        res = asyncio.run(mcp_map("https://example.com", limit=50))
+        assert res == ["https://example.com/p1", "https://example.com/p2"]
+        mock_disc.assert_awaited_once_with("https://example.com", limit=50, timeout=15)

{webget_cli-0.9.0 → webget_cli-0.11.0}/tests/test_mcp_profile.py

@@ -79,7 +79,7 @@ def test_mcp_fetch_with_profile_uses_session(tmp_path):
         ok, anon = asyncio.run(asyncio.wait_for(run(), timeout=60))
         ok_text = ok.content[0].text if ok.content else ""
         anon_text = anon.content[0].text if anon.content else ""
-        assert not ok.is_error, ok_text
+        assert not ok.isError, ok_text
         assert '"status":"success"' in ok_text, ok_text
         assert "you are authenticated" in ok_text, ok_text
         assert '"authenticated":true' in ok_text, ok_text  # session was used
@@ -105,7 +105,7 @@ def test_mcp_fetch_unknown_profile_is_hard_error(tmp_path):
             return res
 
     res = asyncio.run(asyncio.wait_for(run(), timeout=60))
-    assert not res.is_error
+    assert not res.isError
     assert "profile 'ghost' not found" in res.content[0].text
 
 
@@ -150,7 +150,7 @@ def test_mcp_fetch_enriched_output_present(tmp_path):
         anon_text = anon.content[0].text if anon.content else ""
 
         # Authenticated: standard assertions
-        assert not ok.is_error, ok_text
+        assert not ok.isError, ok_text
         assert '"status":"success"' in ok_text
         # auth
         assert '"auth"' in ok_text

{webget_cli-0.9.0 → webget_cli-0.11.0}/tests/test_mcp_server.py

@@ -31,7 +31,7 @@ def test_tools_listed():
 
     # order is not a contract; membership is
     assert sorted(_run(run())) == sorted(
-        ["search", "fetch", "search_fetch", "list_profiles", "login"]
+        ["search", "fetch", "search_fetch", "list_profiles", "login", "map"]
     )
 
 
@@ -53,7 +53,7 @@ def test_invalid_strategy_returns_error_not_crash():
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "error" in res.content[0].text
 
 
@@ -72,7 +72,7 @@ def test_firecrawl_without_key_returns_error_not_crash():
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "error" in res.content[0].text
 
 
@@ -95,7 +95,7 @@ def test_server_stays_alive_after_bad_calls():
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "success" in res.content[0].text
 
 
@@ -118,7 +118,7 @@ def test_fetch_invalid_profile_returns_error_not_crash():
             return res, ok
 
     res, ok = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "invalid profile name" in res.content[0].text
     assert "success" in ok.content[0].text  # server alive after the bad call
 
@@ -139,7 +139,7 @@ def test_fetch_nonexistent_profile_returns_error():
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "profile 'ghost' not found" in res.content[0].text
 
 
@@ -161,9 +161,9 @@ def test_search_fetch_invalid_profile_returns_error_not_crash():
             return res, ok
 
     res, ok = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "invalid profile name" in res.content[0].text
-    assert not ok.is_error  # server alive after the bad call
+    assert not ok.isError  # server alive after the bad call
 
 
 def test_list_profiles_tool_metadata_only(tmp_path):
@@ -189,7 +189,7 @@ def test_list_profiles_tool_metadata_only(tmp_path):
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     text = res.content[0].text
     assert "sion" in text
     assert "SUPERSECRET" not in text  # cookie values never exposed

webget_cli-0.11.0/tests/test_ssrf_dual_dns.py

@@ -0,0 +1,34 @@
+import socket
+from unittest.mock import patch
+
+from webget.ssrf import _hostname_private, _resolve_hostname_ips
+
+
+def test_resolve_hostname_ips_primary_success():
+    ips = _resolve_hostname_ips("localhost")
+    assert any(ip.startswith("127.") or ip == "::1" for ip in ips)
+
+
+def test_resolve_hostname_ips_fallback_on_primary_failure():
+    orig_getaddrinfo = socket.getaddrinfo
+
+    def mock_getaddrinfo(host, port, *args, **kwargs):
+        if host == "flaky.example":
+            raise socket.gaierror(socket.EAI_NONAME, "Name or service not known")
+        return orig_getaddrinfo(host, port, *args, **kwargs)
+
+    # Secondary resolver gives fallback IP
+    with (
+        patch("socket.getaddrinfo", side_effect=mock_getaddrinfo),
+        patch("webget.ssrf._doh_resolve", return_value=["93.184.216.34"]),
+    ):
+        ips = _resolve_hostname_ips("flaky.example")
+        assert "93.184.216.34" in ips
+
+
+def test_hostname_private_uses_fallback_and_detects_private():
+    with (
+        patch("socket.getaddrinfo", side_effect=socket.gaierror(socket.EAI_NONAME, "Fail")),
+        patch("webget.ssrf._doh_resolve", return_value=["192.168.1.1"]),
+    ):
+        assert _hostname_private("router.local") is True

{webget_cli-0.9.0 → webget_cli-0.11.0}/tests/test_webget.py

@@ -22,12 +22,21 @@ def auth_state(md="", html="", status=None, profile=None):
 
 class TestParseOpts:
     def test_positional(self):
-        remaining, *_, limit, strategy, profile, no_cache, headless, concurrency = opts(
-            "u", "https://x.com"
-        )
+        (
+            remaining,
+            *_,
+            limit,
+            strategy,
+            profile,
+            no_cache,
+            headless,
+            concurrency,
+            retry_transient,
+        ) = opts("u", "https://x.com")
         assert remaining == ["u", "https://x.com"]
         assert limit is None and strategy == "auto" and profile is None
         assert no_cache is False and headless is False and concurrency is None
+        assert retry_transient is False
 
     def test_cookies_short_and_long(self, tmp_path):
         ck = tmp_path / "ck.txt"
@@ -45,27 +54,31 @@ class TestParseOpts:
         assert mc1 == 500 and mc2 == 500
 
     def test_limit(self):
-        *_, limit, _, _, _, _, _ = opts("s", "q", "--limit", "7")
+        *_, limit, _, _, _, _, _, _ = opts("s", "q", "--limit", "7")
         assert limit == 7
 
     def test_profile_and_no_cache(self):
-        *_, profile, no_cache, _, _ = opts("u", "https://x.com", "--profile", "campus")
+        *_, profile, no_cache, _, _, _ = opts("u", "https://x.com", "--profile", "campus")
         assert profile == "campus" and no_cache is False
-        *_, profile, no_cache, _, _ = opts("u", "https://x.com", "--no-cache")
+        *_, profile, no_cache, _, _, _ = opts("u", "https://x.com", "--no-cache")
         assert profile is None and no_cache is True
 
     def test_strategy(self):
-        *_, strategy, _, _, _, _ = opts("u", "https://x.com", "--strategy", "crawl4ai")
+        *_, strategy, _, _, _, _, _ = opts("u", "https://x.com", "--strategy", "crawl4ai")
         assert strategy == "crawl4ai"
 
     def test_concurrency(self):
-        *_, concurrency = opts("u", "https://x.com", "--concurrency", "5")
+        *_, concurrency, _ = opts("u", "https://x.com", "--concurrency", "5")
         assert concurrency == 5
 
     def test_headless(self):
-        *_, headless, _ = opts("login", "https://x.com", "--headless")
+        *_, headless, _, _ = opts("login", "https://x.com", "--headless")
         assert headless is True
 
+    def test_retry_flag(self):
+        *_, retry_transient = opts("u", "https://x.com", "--retry")
+        assert retry_transient is True
+
     def test_unknown_flag_passthrough(self):
         remaining, *_ = opts("u", "https://x.com", "--weird")
         assert "--weird" in remaining
@@ -533,7 +546,9 @@ class TestStrategyMemory:
         webget._learn_strategy("old.com", "crawl4ai")
         # Age the entry beyond the TTL by shifting the clock forward.
         real_time = webget.time.time
-        monkeypatch.setattr(webget.time, "time", lambda: real_time() + webget._STRATEGY_MEMORY_TTL + 1)
+        monkeypatch.setattr(
+            webget.time, "time", lambda: real_time() + webget._STRATEGY_MEMORY_TTL + 1
+        )
         assert webget._load_strategy_memory() == {}
 
     def test_fresh_entry_survives(self, isolated_env):

webget_cli-0.11.0/webget/__init__.py

@@ -0,0 +1,149 @@
+"""webget - local search + scrape, zero API keys, unlimited usage.
+
+A package refactor of the original single-file webget_cli.py. Public API
+is re-exported here so callers can do `from webget import fetch_http,
+scrape_many, ...` instead of importing internal modules.
+
+Sub-modules:
+  - cache     - disk cache + cookie/header parsing
+  - ssrf      - private-IP guard for HTTP + browser paths
+  - profile   - persistent browser-profile (auth session) management
+  - http      - fast-path HTTP fetch + markdown extraction
+  - firecrawl - optional cloud escape-hatch fetch
+  - search    - DuckDuckGo text search + atomic JSON helpers
+  - ladder    - HTTP -> Crawl4AI -> Firecrawl orchestration
+  - cli       - argparse entry point
+
+The legacy single-file import `import webget_cli as webget` keeps
+working via the compatibility shim at ./webget_cli.py.
+"""
+
+from __future__ import annotations
+
+from .cache import (
+    _CACHE_SWEEP_TTL,
+    CACHE_DIR,
+    _cache_path,
+    cache_get,
+    cache_put,
+    parse_cookie_file,
+    parse_headers,
+)
+from .cli import main, parse_opts
+from .discovery import discover_urls
+from .firecrawl import fetch_firecrawl, firecrawl_key
+from .http import MAX_RESPONSE_BYTES, ResponseTooLarge, _extract_markdown, fetch_http
+from .ladder import (
+    _DEFAULT_CONCURRENCY,
+    _STRATEGY_MEMORY_TTL,
+    _auth_message,
+    _crawl4ai_once,
+    _ladder,
+    _learn_strategy,
+    _load_strategy_memory,
+    _normalize_hit,
+    _reorder_steps_by_domain,
+    _save_strategy_memory,
+    _strategy_memory_path,
+    _terminal_state,
+    scrape_many,
+)
+from .profile import (
+    _PROFILE_NAME_RE,
+    PROFILE_DIR,
+    _auth_state,
+    _cookie_belongs_to,
+    _domain_match,
+    _effective_cookies,
+    _fmt_age,
+    _login_flow,
+    _logout_domain_regex,
+    _logout_flow,
+    _profile_meta,
+    _profile_root,
+    _prune_storage_cookies,
+    _valid_site_url,
+    _wait_for_session_cookies,
+    _warn,
+    list_profiles,
+    load_profile_cookies,
+    profile_dir,
+    profile_exists,
+    profile_state_path,
+)
+from .search import _read_json, _write_json, search
+from .ssrf import (
+    SSRFError,
+    _guard_browser_routes,
+    _hostname_private,
+    _ip_is_private,
+    _is_private_target,
+    _private_ip_for,
+    _request_body_bytes,
+    _resolve_hostname_ips,
+)
+
+# Sorted to satisfy ruff RUF022; module grouping lives in the imports above.
+__all__ = [
+    "CACHE_DIR",
+    "MAX_RESPONSE_BYTES",
+    "PROFILE_DIR",
+    "_CACHE_SWEEP_TTL",
+    "_DEFAULT_CONCURRENCY",
+    "_PROFILE_NAME_RE",
+    "_STRATEGY_MEMORY_TTL",
+    "ResponseTooLarge",
+    "SSRFError",
+    "_auth_message",
+    "_auth_state",
+    "_cache_path",
+    "_cookie_belongs_to",
+    "_crawl4ai_once",
+    "_domain_match",
+    "_effective_cookies",
+    "_extract_markdown",
+    "_fmt_age",
+    "_guard_browser_routes",
+    "_hostname_private",
+    "_ip_is_private",
+    "_is_private_target",
+    "_ladder",
+    "_learn_strategy",
+    "_load_strategy_memory",
+    "_login_flow",
+    "_logout_domain_regex",
+    "_logout_flow",
+    "_normalize_hit",
+    "_private_ip_for",
+    "_profile_meta",
+    "_profile_root",
+    "_prune_storage_cookies",
+    "_read_json",
+    "_reorder_steps_by_domain",
+    "_request_body_bytes",
+    "_resolve_hostname_ips",
+    "_save_strategy_memory",
+    "_strategy_memory_path",
+    "_terminal_state",
+    "_valid_site_url",
+    "_wait_for_session_cookies",
+    "_warn",
+    "_write_json",
+    "cache_get",
+    "cache_put",
+    "discover_urls",
+    "fetch_firecrawl",
+    "fetch_http",
+    "firecrawl_key",
+    "list_profiles",
+    "load_profile_cookies",
+    "main",
+    "parse_cookie_file",
+    "parse_headers",
+    "parse_opts",
+    "profile_dir",
+    "profile_exists",
+    "profile_state_path",
+    "scrape_many",
+    "search",
+]