webget-cli 0.10.0__tar.gz → 0.11.0__tar.gz

{webget_cli-0.10.0 → webget_cli-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.10.0
+Version: 0.11.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0

{webget_cli-0.10.0 → webget_cli-0.11.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.10.0"
+version = "0.11.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"

webget_cli-0.11.0/tests/test_discovery_map.py

@@ -0,0 +1,29 @@
+import asyncio
+
+from webget.discovery import _extract_sitemap_urls, discover_urls
+
+
+def test_extract_sitemap_urls_standard_xml():
+    xml = """<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+            <loc>https://example.com/page1</loc>
+        </url>
+        <url>
+            <loc>https://example.com/page2</loc>
+        </url>
+    </urlset>
+    """
+    urls = _extract_sitemap_urls(xml)
+    assert urls == ["https://example.com/page1", "https://example.com/page2"]
+
+
+def test_extract_sitemap_urls_malformed_regex_fallback():
+    xml = """<urlset><url><loc>https://example.com/broken1</loc></unclosed>"""
+    urls = _extract_sitemap_urls(xml)
+    assert "https://example.com/broken1" in urls
+
+
+def test_discover_urls_private_target_blocked():
+    urls = asyncio.run(discover_urls("http://127.0.0.1/sitemap.xml", allow_private=False))
+    assert urls == []

webget_cli-0.11.0/tests/test_ladder_retry.py

@@ -0,0 +1,41 @@
+import asyncio
+from unittest.mock import patch
+
+from webget.ladder import scrape_many
+
+
+def test_scrape_many_retry_transient_timeout(fresh_cache, server):
+    calls = []
+
+    async def mock_fetch(url, *args, **kwargs):
+        calls.append(url)
+        if len(calls) == 1:
+            raise TimeoutError("timeout")
+        return {
+            "title": "Success After Retry",
+            "markdown": "Valid content length " * 10,
+            "status": "success",
+        }
+
+    async def run_without_retry():
+        return await scrape_many(
+            [server.url("/test")], strategy="http", per_url_timeout=1, retry_transient=False
+        )
+
+    async def run_with_retry():
+        return await scrape_many(
+            [server.url("/test")], strategy="http", per_url_timeout=1, retry_transient=True
+        )
+
+    with patch("webget.ladder._resolve_fetch_http", return_value=mock_fetch):
+        # Without retry flag -> error on first timeout
+        res = asyncio.run(run_without_retry())
+        target = server.url("/test")
+        assert res[target]["status"] == "error"
+        assert res[target]["attempts"] == 1
+
+        # With retry_transient=True -> retries and succeeds on 2nd attempt
+        calls.clear()
+        res = asyncio.run(run_with_retry())
+        assert res[target]["status"] == "success"
+        assert res[target]["attempts"] == 2

{webget_cli-0.10.0 → webget_cli-0.11.0}/tests/test_login_flow.py

@@ -102,7 +102,9 @@ def _spawn_mcp(profile_root):
         "WEBGET_PROFILE_DIR": str(profile_root),
         "WEBGET_ALLOW_PRIVATE": "1",
     }
-    return StdioServerParameters(command=sys.executable, args=[str(ROOT / "webget_mcp.py")], env=env)
+    return StdioServerParameters(
+        command=sys.executable, args=[str(ROOT / "webget_mcp.py")], env=env
+    )
 
 
 def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
@@ -117,14 +119,18 @@ def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
     gated_url = server.url("/cookie-gated")
 
     async def run():
-        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+        async with (
+            stdio_client(_spawn_mcp(root)) as (read, write),
+            ClientSession(read, write) as session,
+        ):
             await session.initialize()
             res = await session.call_tool(
                 "login",
                 {"url": login_url, "profile": "mcplogin", "headless": True, "wait_seconds": 30},
             )
             authed = await session.call_tool(
-                "fetch", {"url": gated_url, "strategy": "http", "no_cache": True, "profile": "mcplogin"}
+                "fetch",
+                {"url": gated_url, "strategy": "http", "no_cache": True, "profile": "mcplogin"},
             )
             anon = await session.call_tool(
                 "fetch", {"url": gated_url, "strategy": "http", "no_cache": True}
@@ -155,16 +161,21 @@ def test_mcp_login_rejects_bad_url_and_profile(server, tmp_path):
     root = _profile_root(tmp_path)
 
     async def run():
-        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+        async with (
+            stdio_client(_spawn_mcp(root)) as (read, write),
+            ClientSession(read, write) as session,
+        ):
             await session.initialize()
             bad_url = await session.call_tool(
                 "login", {"url": "not a url", "profile": "x", "headless": True, "wait_seconds": 5}
             )
             bad_name = await session.call_tool(
-                "login", {"url": server.url("/"), "profile": "../evil", "headless": True, "wait_seconds": 5}
+                "login",
+                {"url": server.url("/"), "profile": "../evil", "headless": True, "wait_seconds": 5},
             )
             bad_secs = await session.call_tool(
-                "login", {"url": server.url("/"), "profile": "x", "headless": True, "wait_seconds": 1}
+                "login",
+                {"url": server.url("/"), "profile": "x", "headless": True, "wait_seconds": 1},
             )
             return bad_url, bad_name, bad_secs
 

webget_cli-0.11.0/tests/test_mcp_map.py

@@ -0,0 +1,18 @@
+import asyncio
+from unittest.mock import AsyncMock, patch
+
+from webget_mcp import map as mcp_map
+
+
+def test_mcp_map_clamps_limit():
+    res = asyncio.run(mcp_map("https://example.com", limit=-5))
+    assert len(res) == 1
+    assert "error: limit must be between" in res[0]
+
+
+def test_mcp_map_calls_discover_urls():
+    with patch("webget_cli.discover_urls", new_callable=AsyncMock) as mock_disc:
+        mock_disc.return_value = ["https://example.com/p1", "https://example.com/p2"]
+        res = asyncio.run(mcp_map("https://example.com", limit=50))
+        assert res == ["https://example.com/p1", "https://example.com/p2"]
+        mock_disc.assert_awaited_once_with("https://example.com", limit=50, timeout=15)

{webget_cli-0.10.0 → webget_cli-0.11.0}/tests/test_mcp_server.py

@@ -31,7 +31,7 @@ def test_tools_listed():
 
     # order is not a contract; membership is
     assert sorted(_run(run())) == sorted(
-        ["search", "fetch", "search_fetch", "list_profiles", "login"]
+        ["search", "fetch", "search_fetch", "list_profiles", "login", "map"]
     )
 
 

webget_cli-0.11.0/tests/test_ssrf_dual_dns.py

@@ -0,0 +1,34 @@
+import socket
+from unittest.mock import patch
+
+from webget.ssrf import _hostname_private, _resolve_hostname_ips
+
+
+def test_resolve_hostname_ips_primary_success():
+    ips = _resolve_hostname_ips("localhost")
+    assert any(ip.startswith("127.") or ip == "::1" for ip in ips)
+
+
+def test_resolve_hostname_ips_fallback_on_primary_failure():
+    orig_getaddrinfo = socket.getaddrinfo
+
+    def mock_getaddrinfo(host, port, *args, **kwargs):
+        if host == "flaky.example":
+            raise socket.gaierror(socket.EAI_NONAME, "Name or service not known")
+        return orig_getaddrinfo(host, port, *args, **kwargs)
+
+    # Secondary resolver gives fallback IP
+    with (
+        patch("socket.getaddrinfo", side_effect=mock_getaddrinfo),
+        patch("webget.ssrf._doh_resolve", return_value=["93.184.216.34"]),
+    ):
+        ips = _resolve_hostname_ips("flaky.example")
+        assert "93.184.216.34" in ips
+
+
+def test_hostname_private_uses_fallback_and_detects_private():
+    with (
+        patch("socket.getaddrinfo", side_effect=socket.gaierror(socket.EAI_NONAME, "Fail")),
+        patch("webget.ssrf._doh_resolve", return_value=["192.168.1.1"]),
+    ):
+        assert _hostname_private("router.local") is True

{webget_cli-0.10.0 → webget_cli-0.11.0}/tests/test_webget.py

@@ -22,12 +22,21 @@ def auth_state(md="", html="", status=None, profile=None):
 
 class TestParseOpts:
     def test_positional(self):
-        remaining, *_, limit, strategy, profile, no_cache, headless, concurrency = opts(
-            "u", "https://x.com"
-        )
+        (
+            remaining,
+            *_,
+            limit,
+            strategy,
+            profile,
+            no_cache,
+            headless,
+            concurrency,
+            retry_transient,
+        ) = opts("u", "https://x.com")
         assert remaining == ["u", "https://x.com"]
         assert limit is None and strategy == "auto" and profile is None
         assert no_cache is False and headless is False and concurrency is None
+        assert retry_transient is False
 
     def test_cookies_short_and_long(self, tmp_path):
         ck = tmp_path / "ck.txt"
@@ -45,27 +54,31 @@ class TestParseOpts:
         assert mc1 == 500 and mc2 == 500
 
     def test_limit(self):
-        *_, limit, _, _, _, _, _ = opts("s", "q", "--limit", "7")
+        *_, limit, _, _, _, _, _, _ = opts("s", "q", "--limit", "7")
         assert limit == 7
 
     def test_profile_and_no_cache(self):
-        *_, profile, no_cache, _, _ = opts("u", "https://x.com", "--profile", "campus")
+        *_, profile, no_cache, _, _, _ = opts("u", "https://x.com", "--profile", "campus")
         assert profile == "campus" and no_cache is False
-        *_, profile, no_cache, _, _ = opts("u", "https://x.com", "--no-cache")
+        *_, profile, no_cache, _, _, _ = opts("u", "https://x.com", "--no-cache")
         assert profile is None and no_cache is True
 
     def test_strategy(self):
-        *_, strategy, _, _, _, _ = opts("u", "https://x.com", "--strategy", "crawl4ai")
+        *_, strategy, _, _, _, _, _ = opts("u", "https://x.com", "--strategy", "crawl4ai")
         assert strategy == "crawl4ai"
 
     def test_concurrency(self):
-        *_, concurrency = opts("u", "https://x.com", "--concurrency", "5")
+        *_, concurrency, _ = opts("u", "https://x.com", "--concurrency", "5")
         assert concurrency == 5
 
     def test_headless(self):
-        *_, headless, _ = opts("login", "https://x.com", "--headless")
+        *_, headless, _, _ = opts("login", "https://x.com", "--headless")
         assert headless is True
 
+    def test_retry_flag(self):
+        *_, retry_transient = opts("u", "https://x.com", "--retry")
+        assert retry_transient is True
+
     def test_unknown_flag_passthrough(self):
         remaining, *_ = opts("u", "https://x.com", "--weird")
         assert "--weird" in remaining
@@ -533,7 +546,9 @@ class TestStrategyMemory:
         webget._learn_strategy("old.com", "crawl4ai")
         # Age the entry beyond the TTL by shifting the clock forward.
         real_time = webget.time.time
-        monkeypatch.setattr(webget.time, "time", lambda: real_time() + webget._STRATEGY_MEMORY_TTL + 1)
+        monkeypatch.setattr(
+            webget.time, "time", lambda: real_time() + webget._STRATEGY_MEMORY_TTL + 1
+        )
         assert webget._load_strategy_memory() == {}
 
     def test_fresh_entry_survives(self, isolated_env):

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget/__init__.py

@@ -17,6 +17,7 @@ Sub-modules:
 The legacy single-file import `import webget_cli as webget` keeps
 working via the compatibility shim at ./webget_cli.py.
 """
+
 from __future__ import annotations
 
 from .cache import (
@@ -29,6 +30,7 @@ from .cache import (
     parse_headers,
 )
 from .cli import main, parse_opts
+from .discovery import discover_urls
 from .firecrawl import fetch_firecrawl, firecrawl_key
 from .http import MAX_RESPONSE_BYTES, ResponseTooLarge, _extract_markdown, fetch_http
 from .ladder import (
@@ -78,6 +80,7 @@ from .ssrf import (
     _is_private_target,
     _private_ip_for,
     _request_body_bytes,
+    _resolve_hostname_ips,
 )
 
 # Sorted to satisfy ruff RUF022; module grouping lives in the imports above.
@@ -118,6 +121,7 @@ __all__ = [
     "_read_json",
     "_reorder_steps_by_domain",
     "_request_body_bytes",
+    "_resolve_hostname_ips",
     "_save_strategy_memory",
     "_strategy_memory_path",
     "_terminal_state",
@@ -127,6 +131,7 @@ __all__ = [
     "_write_json",
     "cache_get",
     "cache_put",
+    "discover_urls",
     "fetch_firecrawl",
     "fetch_http",
     "firecrawl_key",
@@ -141,4 +146,4 @@ __all__ = [
     "profile_state_path",
     "scrape_many",
     "search",
-]
+]

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget/cache.py

@@ -12,6 +12,7 @@ in the test suite: the canonical location for the path is the shim's
 module attribute, and package sub-modules follow whatever the shim
 currently exposes.
 """
+
 from __future__ import annotations
 
 import hashlib
@@ -184,4 +185,4 @@ def cache_put(url, cookies, headers, max_chars, data, profile=None):
             try:
                 os.remove(tmp)
             except OSError:
-                pass
+                pass

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget/cli.py

@@ -9,6 +9,7 @@ Usage (from --help):
   webget login URL --profile X   Open browser, log in manually, persist session
   webget profiles [--json]       List profiles and session status
   webget logout URL --profile X  Clear auth for one domain, keep the rest
+  webget map URL [--limit N]     Discover URLs via sitemaps and robots.txt
 Aliases: search = s, fetch = u, search-fetch = su
 
 Options:
@@ -34,6 +35,7 @@ Session management:
   webget login never stores passwords and never fills forms. You log in
   yourself in the opened browser window; webget just persists the session.
 """
+
 from __future__ import annotations
 
 import asyncio
@@ -56,17 +58,18 @@ from .profile import (
 from .search import search
 
 __doc__ = (
-    'webget - local search + scrape, zero API keys, unlimited usage.\n'
-    'Usage:\n'
+    "webget - local search + scrape, zero API keys, unlimited usage.\n"
+    "Usage:\n"
     '  webget s "query" [n]           Search via DuckDuckGo (default 5)\n'
     '  webget u "https://..."         Scrape URL -> markdown (HTTP fast path, falls back)\n'
     '  webget su "query" [n]          Search + scrape top n results (default 3, parallel)\n'
     '  webget s "q" | webget u -      Pipe: pass URL from search via stdin\n'
-    '                                 (multi-line stdin = batch scrape)\n'
-    '  webget login URL --profile X   Open browser, log in manually, persist session\n'
-    '  webget profiles [--json]       List profiles and session status\n'
-    '  webget logout URL --profile X  Clear auth for one domain, keep the rest\n'
-    'Aliases: search = s, fetch = u, search-fetch = su\n'
+    "                                 (multi-line stdin = batch scrape)\n"
+    "  webget login URL --profile X   Open browser, log in manually, persist session\n"
+    "  webget profiles [--json]       List profiles and session status\n"
+    "  webget logout URL --profile X  Clear auth for one domain, keep the rest\n"
+    "  webget map URL [--limit N]     Discover URLs via sitemaps and robots.txt\n"
+    "Aliases: search = s, fetch = u, search-fetch = su\n"
 )
 
 
@@ -85,6 +88,7 @@ def parse_opts(args):
     no_cache = False
     headless = False
     concurrency = None
+    retry_transient = False
     remaining = []
     i = 0
     while i < len(args):
@@ -97,6 +101,9 @@ def parse_opts(args):
         elif args[i] == "--concurrency" and i + 1 < len(args):
             concurrency = int(args[i + 1])
             i += 2
+        elif args[i] in ("-r", "--retry", "--retry-transient"):
+            retry_transient = True
+            i += 1
         elif args[i] == "--no-cache":
             no_cache = True
             i += 1
@@ -145,6 +152,7 @@ def parse_opts(args):
         no_cache,
         headless,
         concurrency,
+        retry_transient,
     )
 
 
@@ -228,6 +236,7 @@ def main():
         no_cache,
         headless,
         concurrency,
+        retry_transient,
     ) = parse_opts(args)
 
     if concurrency is not None and concurrency < 1:
@@ -260,6 +269,26 @@ def main():
         return
     elif cmd == "logout":
         sys.exit(cmd_logout(q, profile))
+    elif cmd == "map":
+        from .discovery import discover_urls
+
+        n = limit or 100
+        timeout = timeout_override or 10
+        urls = asyncio.run(
+            discover_urls(
+                q,
+                limit=n,
+                timeout=timeout,
+                headers=headers,
+                allow_private=no_cache,
+            )
+        )
+        if json_out:
+            print(json.dumps(urls, indent=2))
+            return
+        for u in urls:
+            print(u)
+        return
 
     if cmd == "s":
         n = limit or (int(args[2]) if len(args) > 2 else 5)
@@ -291,6 +320,7 @@ def main():
                 profile=profile,
                 no_cache=no_cache,
                 max_concurrency=concurrency,
+                retry_transient=retry_transient,
             )
         )
         if json_out:
@@ -374,4 +404,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

webget_cli-0.11.0/webget/discovery.py

@@ -0,0 +1,94 @@
+import re
+import xml.etree.ElementTree as ET
+from urllib.parse import urljoin, urlparse
+
+import httpx
+
+from .ssrf import _is_private_target
+
+
+def _extract_sitemap_urls(xml_content):
+    urls = []
+    try:
+        root = ET.fromstring(xml_content)
+        # Handle namespaces like {http://www.sitemaps.org/schemas/sitemap/0.9}loc
+        for elem in root.iter():
+            if elem.tag.endswith("loc") and elem.text:
+                text = elem.text.strip()
+                if text.startswith(("http://", "https://")):
+                    urls.append(text)
+    except Exception:  # noqa: BLE001
+        # Simple regex fallback if malformed XML
+        matches = re.findall(r"<loc>\s*(https?://[^\s<]+)\s*</loc>", xml_content, re.IGNORECASE)
+        urls.extend(matches)
+    return urls
+
+
+async def discover_urls(
+    target_url,
+    limit=100,
+    timeout=10,
+    headers=None,
+    allow_private=None,
+):
+    """Discover URLs for a domain by checking standard sitemap endpoints and robots.txt.
+    Returns list of URLs bounded by limit.
+    """
+    if _is_private_target(target_url, allow_private=allow_private):
+        return []
+
+    parsed = urlparse(target_url)
+    if not parsed.scheme or not parsed.netloc:
+        return []
+
+    base_origin = f"{parsed.scheme}://{parsed.netloc}"
+    sitemap_candidates = [
+        urljoin(base_origin, "/sitemap.xml"),
+        urljoin(base_origin, "/sitemap_index.xml"),
+        urljoin(base_origin, "/sitemap/sitemap.xml"),
+    ]
+
+    discovered = set()
+    req_headers = {"User-Agent": "webget/discovery"}
+    if headers:
+        req_headers.update(headers)
+
+    async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+        # First check robots.txt for custom Sitemap directives
+        try:
+            robots_resp = await client.get(urljoin(base_origin, "/robots.txt"), headers=req_headers)
+            if robots_resp.status_code == 200:
+                for line in robots_resp.text.splitlines():
+                    if line.strip().lower().startswith("sitemap:"):
+                        sm = line.split(":", 1)[1].strip()
+                        if sm.startswith(("http://", "https://")):
+                            sitemap_candidates.insert(0, sm)
+        except Exception:  # noqa: BLE001, S110
+            pass
+
+        # Check sitemaps
+        for sm_url in sitemap_candidates:
+            if len(discovered) >= limit:
+                break
+            if _is_private_target(sm_url, allow_private=allow_private):
+                continue
+            try:
+                resp = await client.get(sm_url, headers=req_headers)
+                if resp.status_code == 200 and resp.text:
+                    found = _extract_sitemap_urls(resp.text)
+                    for u in found:
+                        # Check sub-sitemaps if any
+                        if (
+                            (u.endswith(".xml") or "sitemap" in u)
+                            and u not in sitemap_candidates
+                            and len(sitemap_candidates) < 10
+                        ):
+                            sitemap_candidates.append(u)
+                        else:
+                            discovered.add(u)
+                            if len(discovered) >= limit:
+                                break
+            except Exception:  # noqa: BLE001, S112
+                continue
+
+    return sorted(discovered)[:limit]

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget/firecrawl.py

@@ -8,6 +8,7 @@ hosts/IPs. webget guarantees only that a private/internal URL is
 never SENT to Firecrawl (pre-check in scrape_many blocks it before the
 ladder runs) and that Firecrawl is strictly opt-in.
 """
+
 from __future__ import annotations
 
 import os
@@ -57,4 +58,4 @@ async def fetch_firecrawl(url, max_chars, key, timeout=30):
             "markdown": md[:max_chars],
             "status_code": r.status_code,
             "html": "",
-        }
+        }

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget/http.py

@@ -5,6 +5,7 @@ following so the SSRF guard runs on every hop, and a streaming body cap
 so a giant/binary download cannot exhaust memory. Extraction tries
 trafilatura first (clean article text), then markdownify as a fallback.
 """
+
 from __future__ import annotations
 
 import asyncio
@@ -93,9 +94,7 @@ async def fetch_http(url, max_chars, cookies=None, headers=None, timeout=15):
             except ImportError:
                 _fn = None
             resolver = _fn if (_fn is not None and _fn is not _private_ip_for) else _private_ip_for
-            ip = await asyncio.wait_for(
-                asyncio.to_thread(resolver, target), max(remaining, 0.1)
-            )
+            ip = await asyncio.wait_for(asyncio.to_thread(resolver, target), max(remaining, 0.1))
         except TimeoutError:
             raise TimeoutError(f"DNS resolution exceeded {timeout}s") from None
         if ip is not None:
@@ -172,4 +171,4 @@ async def fetch_http(url, max_chars, cookies=None, headers=None, timeout=15):
                     "markdown": md[:max_chars],
                     "status_code": r.status_code,
                     "html": html[:8000],
-                }
+                }

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget/ladder.py

@@ -5,6 +5,7 @@ is recorded with reasons and the ladder escalates. Per-domain strategy
 memory promotes a known-good strategy to the front of the ladder so
 subsequent 'auto' runs skip likely-failures.
 """
+
 from __future__ import annotations
 
 import asyncio
@@ -316,6 +317,7 @@ async def scrape_many(
     profile=None,
     no_cache=False,
     max_concurrency=None,
+    retry_transient=False,
 ):
     steps = _ladder(strategy, _firecrawl_module.firecrawl_key())
     # Per-domain strategy memory: when strategy is "auto" and we have
@@ -478,6 +480,39 @@ async def scrape_many(
                 results[url] = out
         pending = [u for u in pending if u not in results]
 
+        # Optional retry pass for transient timeouts on the fast HTTP path
+        if retry_transient and pending and "http" in steps:
+            transient_candidates = [
+                u
+                for u in pending
+                if reasons[u]
+                and reasons[u][-1][0] == "error"
+                and "timeout" in str(reasons[u][-1][2]).lower()
+            ]
+            if transient_candidates:
+                retry_timeout = int(per_url_timeout * 1.5)
+
+                async def http_retry_one(u):
+                    async with sem:
+                        try:
+                            res = await _resolve_fetch_http()(
+                                u,
+                                max_chars,
+                                _effective_cookies(cookies, profile),
+                                headers,
+                                timeout=retry_timeout,
+                            )
+                            return u, await record(u, "http", res=res)
+                        except Exception as e:  # noqa: BLE001
+                            return u, await record(u, "http", exc=e)
+
+                for url, out in await asyncio.gather(
+                    *(http_retry_one(u) for u in transient_candidates)
+                ):
+                    if out:
+                        results[url] = out
+                pending = [u for u in pending if u not in results]
+
     # Pass 2: Crawl4AI browser - only launched if something still needs it.
     if pending and "crawl4ai" in steps:
         try:
@@ -507,7 +542,9 @@ async def scrape_many(
                     async def crawl_one(url):
                         async with sem:
                             try:
-                                res = await _resolve_crawl4ai_once()(crawler_ctx, cfg, url, per_url_timeout)
+                                res = await _resolve_crawl4ai_once()(
+                                    crawler_ctx, cfg, url, per_url_timeout
+                                )
                                 res["markdown"] = res.get("markdown", "")[:max_chars]
                                 return url, await record(url, "crawl4ai", res=res)
                             except TimeoutError:
@@ -573,4 +610,4 @@ async def scrape_many(
             "auth": auth,
             "reasons": [{"state": s, "method": m, "detail": d} for s, m, d in reasons[url]],
         }
-    return results
+    return results

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget/profile.py

@@ -5,6 +5,7 @@ Profiles live at ~/.local/share/webget/profiles/<name> by default
 (cookies + local storage) so authenticated fetches via the HTTP fast
 path can reuse a session captured once in a real browser.
 """
+
 from __future__ import annotations
 
 import asyncio
@@ -406,4 +407,4 @@ async def _logout_flow(site, profile):
         except Exception as e:  # noqa: BLE001 - warn, still done
             _warn(f"could not clear browser cookies for {host}: {e}")
         await context.close()
-    return True, removed
+    return True, removed

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget/search.py

@@ -1,4 +1,5 @@
 """DuckDuckGo search + atomic JSON helpers for webget."""
+
 from __future__ import annotations
 
 import json
@@ -34,4 +35,4 @@ def _write_json(path, data):
         try:
             os.remove(tmp)
         except OSError:
-            pass
+            pass

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget/ssrf.py

@@ -19,6 +19,7 @@ definition and all call sites were in the same module; the package
 layout would otherwise bind the symbol at import time and miss the
 patch.
 """
+
 from __future__ import annotations
 
 import asyncio
@@ -45,6 +46,51 @@ def _ip_is_private(ip):
     )
 
 
+def _doh_resolve(host, timeout=3.0):
+    """Fallback DoH resolver (Cloudflare 1.1.1.1 / Google 8.8.8.8) using httpx.
+    Returns list of IP strings if resolvable, else empty list.
+    """
+    if os.environ.get("WEBGET_DISABLE_DOH") == "1":
+        return []
+    import urllib.parse
+
+    import httpx
+
+    urls = [
+        f"https://1.1.1.1/dns-query?name={urllib.parse.quote(host)}&type=A",
+        f"https://dns.google/resolve?name={urllib.parse.quote(host)}&type=A",
+    ]
+    for url in urls:
+        try:
+            with httpx.Client(timeout=timeout, verify=True) as client:
+                res = client.get(url, headers={"accept": "application/dns-json"})
+                if res.status_code == 200:
+                    data = res.json()
+                    answers = data.get("Answer", [])
+                    ips = [
+                        ans.get("data")
+                        for ans in answers
+                        if ans.get("type") == 1 and ans.get("data")
+                    ]
+                    if ips:
+                        return ips
+        except Exception:  # noqa: BLE001, S112 - DoH is best-effort fallback
+            continue
+    return []
+
+
+def _resolve_hostname_ips(host):
+    """Resolve a hostname to a list of IP address strings.
+    Tries system getaddrinfo first; falls back to DoH on DNS resolution errors.
+    """
+    try:
+        infos = socket.getaddrinfo(host, None)
+        return [i[4][0] for i in infos]
+    except OSError:
+        doh_ips = _doh_resolve(host)
+        return doh_ips
+
+
 def _hostname_private(host):
     """Resolve a hostname once and check every address. Cached per process.
 
@@ -55,13 +101,15 @@ def _hostname_private(host):
     """
     if host in _PRIVATE_IP_CACHE:
         return _PRIVATE_IP_CACHE[host]
-    try:
-        infos = socket.getaddrinfo(host, None)
-    except OSError:
+    ips = _resolve_hostname_ips(host)
+    if not ips:
         # DNS failure is not a privacy violation; let the fetch fail normally.
         _PRIVATE_IP_CACHE[host] = False
         return False
-    private = any(_ip_is_private(ipaddress.ip_address(i[4][0])) for i in infos)
+    try:
+        private = any(_ip_is_private(ipaddress.ip_address(ip)) for ip in ips)
+    except ValueError:
+        private = False
     if len(_PRIVATE_IP_CACHE) < 512:
         _PRIVATE_IP_CACHE[host] = private
     return private
@@ -229,4 +277,4 @@ async def _guard_browser_routes(crawler_ctx):
         # Crawl4AI may create the crawling context lazily on first navigation.
         target.on("context", lambda ctx: asyncio.create_task(register(ctx)))
     else:
-        await register(target)
+        await register(target)

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.10.0
+Version: 0.11.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget_cli.egg-info/SOURCES.txt

@@ -13,20 +13,25 @@ tests/test_auth_review.py
 tests/test_browser_ssrf.py
 tests/test_cache_review.py
 tests/test_concurrency_review.py
+tests/test_discovery_map.py
 tests/test_extraction_markdown.py
 tests/test_firecrawl_policy.py
 tests/test_integration_ladder.py
+tests/test_ladder_retry.py
 tests/test_login_flow.py
 tests/test_mcp_leak_review.py
+tests/test_mcp_map.py
 tests/test_mcp_profile.py
 tests/test_mcp_server.py
 tests/test_mcp_smoke.py
 tests/test_security_review.py
 tests/test_size_review.py
+tests/test_ssrf_dual_dns.py
 tests/test_webget.py
 webget/__init__.py
 webget/cache.py
 webget/cli.py
+webget/discovery.py
 webget/firecrawl.py
 webget/http.py
 webget/ladder.py

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget_cli.py

@@ -18,6 +18,7 @@ If you maintain this shim, keep the export set in sync with webget/
 public API; new internal helpers should be exposed here too if tests
 touch them.
 """
+
 from __future__ import annotations
 
 import time
@@ -32,6 +33,15 @@ from webget import *
 # exactly like before.
 from webget.cli import main
 
+
+def discover_urls(target_url, limit=100, timeout=10, headers=None, allow_private=None):
+    from webget.discovery import discover_urls as _du
+
+    return _du(
+        target_url, limit=limit, timeout=timeout, headers=headers, allow_private=allow_private
+    )
+
+
 __all__ = [
     "CACHE_DIR",
     "MAX_RESPONSE_BYTES",
@@ -45,4 +55,4 @@ __all__ = [
 
 
 if __name__ == "__main__":
-    main()
+    main()

{webget_cli-0.10.0 → webget_cli-0.11.0}/webget_mcp.py

@@ -242,6 +242,28 @@ async def search_fetch(
     return out
 
 
+@mcp.tool()
+async def map(
+    url: str,
+    limit: int = 100,
+    timeout: int = 15,
+) -> list[str]:
+    """Discover URLs under a website by probing standard sitemap endpoints and robots.txt.
+    Returns list of discovered URLs bounded by limit.
+    """
+    for name, value, lo, hi in (
+        ("limit", limit, 1, 1000),
+        ("timeout", timeout, 1, _MAX_TIMEOUT),
+    ):
+        err = _clamp(name, value, lo, hi)
+        if err:
+            return [f"error: {err}"]
+    try:
+        return await wg.discover_urls(url, limit=limit, timeout=timeout)
+    except Exception as e:  # noqa: BLE001
+        return [f"error: {e}"]
+
+
 def main() -> None:
     mcp.run()