webget-cli 0.10.0__tar.gz → 0.12.0__tar.gz

{webget_cli-0.10.0/webget_cli.egg-info → webget_cli-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.10.0
+Version: 0.12.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -23,6 +23,7 @@ Requires-Dist: ddgs
 Requires-Dist: httpx
 Requires-Dist: trafilatura
 Requires-Dist: markdownify
+Requires-Dist: pypdf
 Provides-Extra: browser
 Requires-Dist: crawl4ai>=0.9; extra == "browser"
 Provides-Extra: mcp
@@ -188,13 +189,28 @@ path can reuse them. Secrets are never printed.
       "authenticated": true,
       "state": "success"
     },
+    "metadata": {
+      "author": "Jane Doe",
+      "published_at": "2026-09-01",
+      "site_name": "Campus Portal",
+      "language": "id"
+    },
     "error": null
   }
 }
 ```
 
+`metadata` (author, published date, site name, language) comes from
+trafilatura extraction on the HTTP path; values are `null` when unknown
+or when the winning strategy was not HTTP.
+
 Status values: `success | login_required | challenge | blocked | error`.
 
+The HTTP path also handles non-HTML: JSON becomes a pretty code block,
+CSV becomes a GFM table, RSS/Atom feeds become a link list, PDFs are
+extracted per page (via `pypdf`), plain text passes through. Valid
+non-HTML payloads count as success even when short.
+
 ## Status detection rules
 
 | Signal | State |

{webget_cli-0.10.0 → webget_cli-0.12.0}/README.md

@@ -154,13 +154,28 @@ path can reuse them. Secrets are never printed.
       "authenticated": true,
       "state": "success"
     },
+    "metadata": {
+      "author": "Jane Doe",
+      "published_at": "2026-09-01",
+      "site_name": "Campus Portal",
+      "language": "id"
+    },
     "error": null
   }
 }
 ```
 
+`metadata` (author, published date, site name, language) comes from
+trafilatura extraction on the HTTP path; values are `null` when unknown
+or when the winning strategy was not HTTP.
+
 Status values: `success | login_required | challenge | blocked | error`.
 
+The HTTP path also handles non-HTML: JSON becomes a pretty code block,
+CSV becomes a GFM table, RSS/Atom feeds become a link list, PDFs are
+extracted per page (via `pypdf`), plain text passes through. Valid
+non-HTML payloads count as success even when short.
+
 ## Status detection rules
 
 | Signal | State |

{webget_cli-0.10.0 → webget_cli-0.12.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.10.0"
+version = "0.12.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -26,6 +26,7 @@ dependencies = [
     "httpx",
     "trafilatura",
     "markdownify",
+    "pypdf",
 ]
 
 [project.urls]

webget_cli-0.12.0/tests/test_discovery_map.py

@@ -0,0 +1,29 @@
+import asyncio
+
+from webget.discovery import _extract_sitemap_urls, discover_urls
+
+
+def test_extract_sitemap_urls_standard_xml():
+    xml = """<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+            <loc>https://example.com/page1</loc>
+        </url>
+        <url>
+            <loc>https://example.com/page2</loc>
+        </url>
+    </urlset>
+    """
+    urls = _extract_sitemap_urls(xml)
+    assert urls == ["https://example.com/page1", "https://example.com/page2"]
+
+
+def test_extract_sitemap_urls_malformed_regex_fallback():
+    xml = """<urlset><url><loc>https://example.com/broken1</loc></unclosed>"""
+    urls = _extract_sitemap_urls(xml)
+    assert "https://example.com/broken1" in urls
+
+
+def test_discover_urls_private_target_blocked():
+    urls = asyncio.run(discover_urls("http://127.0.0.1/sitemap.xml", allow_private=False))
+    assert urls == []

webget_cli-0.12.0/tests/test_ladder_retry.py

@@ -0,0 +1,41 @@
+import asyncio
+from unittest.mock import patch
+
+from webget.ladder import scrape_many
+
+
+def test_scrape_many_retry_transient_timeout(fresh_cache, server):
+    calls = []
+
+    async def mock_fetch(url, *args, **kwargs):
+        calls.append(url)
+        if len(calls) == 1:
+            raise TimeoutError("timeout")
+        return {
+            "title": "Success After Retry",
+            "markdown": "Valid content length " * 10,
+            "status": "success",
+        }
+
+    async def run_without_retry():
+        return await scrape_many(
+            [server.url("/test")], strategy="http", per_url_timeout=1, retry_transient=False
+        )
+
+    async def run_with_retry():
+        return await scrape_many(
+            [server.url("/test")], strategy="http", per_url_timeout=1, retry_transient=True
+        )
+
+    with patch("webget.ladder._resolve_fetch_http", return_value=mock_fetch):
+        # Without retry flag -> error on first timeout
+        res = asyncio.run(run_without_retry())
+        target = server.url("/test")
+        assert res[target]["status"] == "error"
+        assert res[target]["attempts"] == 1
+
+        # With retry_transient=True -> retries and succeeds on 2nd attempt
+        calls.clear()
+        res = asyncio.run(run_with_retry())
+        assert res[target]["status"] == "success"
+        assert res[target]["attempts"] == 2

{webget_cli-0.10.0 → webget_cli-0.12.0}/tests/test_login_flow.py

@@ -102,7 +102,9 @@ def _spawn_mcp(profile_root):
         "WEBGET_PROFILE_DIR": str(profile_root),
         "WEBGET_ALLOW_PRIVATE": "1",
     }
-    return StdioServerParameters(command=sys.executable, args=[str(ROOT / "webget_mcp.py")], env=env)
+    return StdioServerParameters(
+        command=sys.executable, args=[str(ROOT / "webget_mcp.py")], env=env
+    )
 
 
 def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
@@ -117,14 +119,18 @@ def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
     gated_url = server.url("/cookie-gated")
 
     async def run():
-        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+        async with (
+            stdio_client(_spawn_mcp(root)) as (read, write),
+            ClientSession(read, write) as session,
+        ):
             await session.initialize()
             res = await session.call_tool(
                 "login",
                 {"url": login_url, "profile": "mcplogin", "headless": True, "wait_seconds": 30},
             )
             authed = await session.call_tool(
-                "fetch", {"url": gated_url, "strategy": "http", "no_cache": True, "profile": "mcplogin"}
+                "fetch",
+                {"url": gated_url, "strategy": "http", "no_cache": True, "profile": "mcplogin"},
             )
             anon = await session.call_tool(
                 "fetch", {"url": gated_url, "strategy": "http", "no_cache": True}
@@ -155,16 +161,21 @@ def test_mcp_login_rejects_bad_url_and_profile(server, tmp_path):
     root = _profile_root(tmp_path)
 
     async def run():
-        async with stdio_client(_spawn_mcp(root)) as (read, write), ClientSession(read, write) as session:
+        async with (
+            stdio_client(_spawn_mcp(root)) as (read, write),
+            ClientSession(read, write) as session,
+        ):
             await session.initialize()
             bad_url = await session.call_tool(
                 "login", {"url": "not a url", "profile": "x", "headless": True, "wait_seconds": 5}
             )
             bad_name = await session.call_tool(
-                "login", {"url": server.url("/"), "profile": "../evil", "headless": True, "wait_seconds": 5}
+                "login",
+                {"url": server.url("/"), "profile": "../evil", "headless": True, "wait_seconds": 5},
             )
             bad_secs = await session.call_tool(
-                "login", {"url": server.url("/"), "profile": "x", "headless": True, "wait_seconds": 1}
+                "login",
+                {"url": server.url("/"), "profile": "x", "headless": True, "wait_seconds": 1},
             )
             return bad_url, bad_name, bad_secs
 

{webget_cli-0.10.0 → webget_cli-0.12.0}/tests/test_mcp_leak_review.py

@@ -24,7 +24,7 @@ LEAK_MARKERS = [
     "/home/",
     "Traceback",
     'File "',
-    "line ",
+    '", line ',  # traceback frames only (plain "line " false-positives on real content, e.g. "online ", "deadline ")
 ]
 
 # The literal variable NAME may legitimately appear in error messages

webget_cli-0.12.0/tests/test_mcp_map.py

@@ -0,0 +1,18 @@
+import asyncio
+from unittest.mock import AsyncMock, patch
+
+from webget_mcp import map as mcp_map
+
+
+def test_mcp_map_clamps_limit():
+    res = asyncio.run(mcp_map("https://example.com", limit=-5))
+    assert len(res) == 1
+    assert "error: limit must be between" in res[0]
+
+
+def test_mcp_map_calls_discover_urls():
+    with patch("webget_cli.discover_urls", new_callable=AsyncMock) as mock_disc:
+        mock_disc.return_value = ["https://example.com/p1", "https://example.com/p2"]
+        res = asyncio.run(mcp_map("https://example.com", limit=50))
+        assert res == ["https://example.com/p1", "https://example.com/p2"]
+        mock_disc.assert_awaited_once_with("https://example.com", limit=50, timeout=15)

{webget_cli-0.10.0 → webget_cli-0.12.0}/tests/test_mcp_server.py

@@ -31,7 +31,7 @@ def test_tools_listed():
 
     # order is not a contract; membership is
     assert sorted(_run(run())) == sorted(
-        ["search", "fetch", "search_fetch", "list_profiles", "login"]
+        ["search", "fetch", "search_fetch", "list_profiles", "login", "map"]
     )
 
 

webget_cli-0.12.0/tests/test_metadata.py

@@ -0,0 +1,129 @@
+"""Ide #1: metadata from trafilatura exposed in fetch results."""
+
+import webget_cli as webget
+
+
+def _rich_html():
+    return (
+        "<html><head><title>T</title>"
+        '<meta name="author" content="Jane Doe">'
+        '<meta property="article:published_time" content="2026-09-01T10:00:00Z">'
+        "</head><body><article><h1>T</h1><p>"
+        + "Lorem ipsum dolor sit amet consectetur adipiscing elit sed do. " * 10
+        + "</p></article></body></html>"
+    )
+
+
+class TestExtractWithMetadata:
+    def test_returns_text_and_metadata(self):
+        text, meta = webget._extract_with_metadata(_rich_html())
+        assert len(text) > 100
+        assert meta["author"] == "Jane Doe"
+        assert "2026-09-01" in (meta.get("published_at") or "")
+
+    def test_fallback_returns_empty_metadata(self, monkeypatch):
+        import trafilatura
+
+        monkeypatch.setattr(trafilatura, "extract", lambda *a, **k: None)
+        body = "<p>" + "Fallback body content here yes indeed. " * 5 + "</p>"
+        text, meta = webget._extract_with_metadata("<h1>H</h1>" + body)
+        assert "# H" in text
+        assert meta == {"author": None, "published_at": None, "site_name": None, "language": None}
+
+    def test_extract_markdown_still_returns_str(self, monkeypatch):
+        # Regression guard: old contract returns plain str, never tuple.
+        import trafilatura
+
+        monkeypatch.setattr(trafilatura, "extract", lambda *a, **k: None)
+        body = "<p>" + "Fallback body content here yes indeed. " * 5 + "</p>"
+        out = webget._extract_markdown("<h1>H</h1>" + body)
+        assert isinstance(out, str)
+        assert "# H" in out
+
+
+class TestFetchHttpMetadata:
+    def test_fetch_http_result_carries_metadata(self, fresh_cache):
+        import asyncio
+
+        res = asyncio.run(webget.fetch_http(fresh_cache.url("/long"), 6000, timeout=10))
+        assert set(res["metadata"]) == {"author", "published_at", "site_name", "language"}
+
+    def test_scrape_many_success_carries_metadata(self, fresh_cache):
+        import asyncio
+
+        out = asyncio.run(webget.scrape_many([fresh_cache.url("/long")], strategy="http"))
+        url = fresh_cache.url("/long")
+        assert out[url]["status"] == "success"
+        assert set(out[url]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }
+
+    def test_cached_hit_carries_metadata(self, fresh_cache):
+        import asyncio
+
+        url = fresh_cache.url("/long")
+        asyncio.run(webget.scrape_many([url], strategy="http"))
+        out = asyncio.run(webget.scrape_many([url], strategy="http"))
+        assert out[url]["cached"] is True
+        assert set(out[url]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }
+
+
+class TestOutputExposure:
+    def test_mcp_fetch_passthrough_has_metadata(self, fresh_cache):
+        import asyncio
+
+        import webget_mcp
+
+        res = asyncio.run(
+            webget_mcp.fetch(fresh_cache.url("/long"), strategy="http", no_cache=True)
+        )
+        assert set(res["metadata"]) == {"author", "published_at", "site_name", "language"}
+
+    def test_mcp_search_fetch_rebuild_has_metadata(self, fresh_cache, monkeypatch):
+        import asyncio
+
+        import webget_mcp
+
+        monkeypatch.setattr(
+            webget_mcp.wg,
+            "search",
+            lambda *a, **k: [{"title": "T", "url": fresh_cache.url("/long"), "snippet": "s"}],
+        )
+        out = asyncio.run(webget_mcp.search_fetch("q", n=1, no_cache=True))
+        assert set(out[0]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }
+
+    def test_cli_su_json_rebuild_has_metadata(self, fresh_cache, monkeypatch, capsys):
+        import webget_cli as wgcli
+        from webget import cli as cli_mod
+
+        monkeypatch.setattr(
+            cli_mod,
+            "search",
+            lambda *a, **k: [{"title": "T", "url": fresh_cache.url("/long"), "snippet": "s"}],
+        )
+        monkeypatch.setattr("sys.argv", ["webget", "su", "q", "1", "--json", "--no-cache"])
+        wgcli.main()
+        import json
+
+        printed = capsys.readouterr().out
+        data = json.loads(printed)
+        url = fresh_cache.url("/long")
+        assert set(data[url]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }

webget_cli-0.12.0/tests/test_nonhtml.py

@@ -0,0 +1,110 @@
+"""Ide #2: non-HTML content routing (JSON/text/CSV/feed/PDF to markdown)."""
+
+import asyncio
+
+import webget_cli as webget
+
+
+class TestJsonRouting:
+    def test_fetch_json_returns_pretty_markdown(self, fresh_cache):
+        url = fresh_cache.url("/json")
+        res = asyncio.run(webget.fetch_http(url, 6000, timeout=10))
+        assert "hello" in res["markdown"]
+        assert "world" in res["markdown"]
+        assert res["metadata"]["site_name"] == "127.0.0.1"
+
+    def test_scrape_many_json_is_success(self, fresh_cache):
+        url = fresh_cache.url("/json")
+        out = asyncio.run(webget.scrape_many([url], strategy="http", no_cache=True))
+        assert out[url]["status"] == "success"
+        assert out[url]["method"] == "http"
+        assert "hello" in out[url]["markdown"]
+
+
+class TestCsvRouting:
+    def test_csv_becomes_gfm_table(self):
+        body = b"name,age\nbudi,25\nsiti,30\n"
+        _title, md, meta = webget._convert_non_html("text/csv", body, "https://example.com/d.csv")
+        assert "| name | age |" in md
+        assert "| budi | 25 |" in md
+        assert meta["site_name"] == "example.com"
+
+    def test_csv_escapes_pipes(self):
+        body = b"a,b\nx|y,z\n"
+        _, md, _ = webget._convert_non_html("text/csv", body, "https://example.com/d.csv")
+        assert "x\\|y" in md
+
+
+class TestFeedRouting:
+    RSS = (
+        b'<?xml version="1.0"?><rss version="2.0"><channel><title>Blog</title>'
+        b"<item><title>Post A</title><link>https://ex.com/a</link>"
+        b"<description>First post here</description></item>"
+        b"<item><title>Post B</title><link>https://ex.com/b</link></item>"
+        b"</channel></rss>"
+    )
+    ATOM = (
+        b'<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom">'
+        b"<title>Blog</title>"
+        b'<entry><title>Entry One</title><link href="https://ex.com/1"/>'
+        b"<summary>Summary one</summary></entry>"
+        b"</feed>"
+    )
+
+    def test_rss_becomes_link_list(self):
+        _title, md, _meta = webget._convert_non_html(
+            "application/rss+xml", self.RSS, "https://ex.com/feed"
+        )
+        assert "[Post A](https://ex.com/a)" in md
+        assert "[Post B](https://ex.com/b)" in md
+        assert "First post here" in md
+
+    def test_atom_becomes_link_list(self):
+        _, md, _ = webget._convert_non_html(
+            "application/atom+xml", self.ATOM, "https://ex.com/feed"
+        )
+        assert "[Entry One](https://ex.com/1)" in md
+
+    def test_xml_without_items_falls_back_to_text(self):
+        _, md, _ = webget._convert_non_html(
+            "application/xml", b"<note><to>u</to></note>", "https://ex.com/n.xml"
+        )
+        assert "to" in md
+
+
+class TestPdfRouting:
+    def test_pdf_routing_uses_pypdf(self, monkeypatch):
+        import sys
+        import types
+
+        fake_page = types.SimpleNamespace(extract_text=lambda: "Hello PDF page one")
+        fake_reader = lambda *a, **k: types.SimpleNamespace(pages=[fake_page])
+        fake_mod = types.SimpleNamespace(PdfReader=fake_reader)
+        monkeypatch.setitem(sys.modules, "pypdf", fake_mod)
+        _title, md, meta = webget._convert_non_html(
+            "application/pdf", b"%PDF-fake", "https://ex.com/d.pdf"
+        )
+        assert "Hello PDF page one" in md
+        assert meta["site_name"] == "ex.com"
+
+    def test_pdf_missing_dep_errors_clearly(self, monkeypatch):
+        import sys
+
+        monkeypatch.setitem(sys.modules, "pypdf", None)
+        import pytest
+
+        with pytest.raises(RuntimeError, match="pypdf"):
+            webget._convert_non_html("application/pdf", b"%PDF-fake", "https://ex.com/d.pdf")
+
+    def test_pdf_empty_text_errors(self, monkeypatch):
+        import sys
+        import types
+
+        fake_page = types.SimpleNamespace(extract_text=lambda: "  ")
+        fake_reader = lambda *a, **k: types.SimpleNamespace(pages=[fake_page])
+        fake_mod = types.SimpleNamespace(PdfReader=fake_reader)
+        monkeypatch.setitem(sys.modules, "pypdf", fake_mod)
+        import pytest
+
+        with pytest.raises(RuntimeError, match="no extractable text"):
+            webget._convert_non_html("application/pdf", b"%PDF-fake", "https://ex.com/d.pdf")

webget_cli-0.12.0/tests/test_ssrf_dual_dns.py

@@ -0,0 +1,34 @@
+import socket
+from unittest.mock import patch
+
+from webget.ssrf import _hostname_private, _resolve_hostname_ips
+
+
+def test_resolve_hostname_ips_primary_success():
+    ips = _resolve_hostname_ips("localhost")
+    assert any(ip.startswith("127.") or ip == "::1" for ip in ips)
+
+
+def test_resolve_hostname_ips_fallback_on_primary_failure():
+    orig_getaddrinfo = socket.getaddrinfo
+
+    def mock_getaddrinfo(host, port, *args, **kwargs):
+        if host == "flaky.example":
+            raise socket.gaierror(socket.EAI_NONAME, "Name or service not known")
+        return orig_getaddrinfo(host, port, *args, **kwargs)
+
+    # Secondary resolver gives fallback IP
+    with (
+        patch("socket.getaddrinfo", side_effect=mock_getaddrinfo),
+        patch("webget.ssrf._doh_resolve", return_value=["93.184.216.34"]),
+    ):
+        ips = _resolve_hostname_ips("flaky.example")
+        assert "93.184.216.34" in ips
+
+
+def test_hostname_private_uses_fallback_and_detects_private():
+    with (
+        patch("socket.getaddrinfo", side_effect=socket.gaierror(socket.EAI_NONAME, "Fail")),
+        patch("webget.ssrf._doh_resolve", return_value=["192.168.1.1"]),
+    ):
+        assert _hostname_private("router.local") is True

{webget_cli-0.10.0 → webget_cli-0.12.0}/tests/test_webget.py

@@ -22,12 +22,21 @@ def auth_state(md="", html="", status=None, profile=None):
 
 class TestParseOpts:
     def test_positional(self):
-        remaining, *_, limit, strategy, profile, no_cache, headless, concurrency = opts(
-            "u", "https://x.com"
-        )
+        (
+            remaining,
+            *_,
+            limit,
+            strategy,
+            profile,
+            no_cache,
+            headless,
+            concurrency,
+            retry_transient,
+        ) = opts("u", "https://x.com")
         assert remaining == ["u", "https://x.com"]
         assert limit is None and strategy == "auto" and profile is None
         assert no_cache is False and headless is False and concurrency is None
+        assert retry_transient is False
 
     def test_cookies_short_and_long(self, tmp_path):
         ck = tmp_path / "ck.txt"
@@ -45,27 +54,31 @@ class TestParseOpts:
         assert mc1 == 500 and mc2 == 500
 
     def test_limit(self):
-        *_, limit, _, _, _, _, _ = opts("s", "q", "--limit", "7")
+        *_, limit, _, _, _, _, _, _ = opts("s", "q", "--limit", "7")
         assert limit == 7
 
     def test_profile_and_no_cache(self):
-        *_, profile, no_cache, _, _ = opts("u", "https://x.com", "--profile", "campus")
+        *_, profile, no_cache, _, _, _ = opts("u", "https://x.com", "--profile", "campus")
         assert profile == "campus" and no_cache is False
-        *_, profile, no_cache, _, _ = opts("u", "https://x.com", "--no-cache")
+        *_, profile, no_cache, _, _, _ = opts("u", "https://x.com", "--no-cache")
         assert profile is None and no_cache is True
 
     def test_strategy(self):
-        *_, strategy, _, _, _, _ = opts("u", "https://x.com", "--strategy", "crawl4ai")
+        *_, strategy, _, _, _, _, _ = opts("u", "https://x.com", "--strategy", "crawl4ai")
         assert strategy == "crawl4ai"
 
     def test_concurrency(self):
-        *_, concurrency = opts("u", "https://x.com", "--concurrency", "5")
+        *_, concurrency, _ = opts("u", "https://x.com", "--concurrency", "5")
         assert concurrency == 5
 
     def test_headless(self):
-        *_, headless, _ = opts("login", "https://x.com", "--headless")
+        *_, headless, _, _ = opts("login", "https://x.com", "--headless")
         assert headless is True
 
+    def test_retry_flag(self):
+        *_, retry_transient = opts("u", "https://x.com", "--retry")
+        assert retry_transient is True
+
     def test_unknown_flag_passthrough(self):
         remaining, *_ = opts("u", "https://x.com", "--weird")
         assert "--weird" in remaining
@@ -533,7 +546,9 @@ class TestStrategyMemory:
         webget._learn_strategy("old.com", "crawl4ai")
         # Age the entry beyond the TTL by shifting the clock forward.
         real_time = webget.time.time
-        monkeypatch.setattr(webget.time, "time", lambda: real_time() + webget._STRATEGY_MEMORY_TTL + 1)
+        monkeypatch.setattr(
+            webget.time, "time", lambda: real_time() + webget._STRATEGY_MEMORY_TTL + 1
+        )
         assert webget._load_strategy_memory() == {}
 
     def test_fresh_entry_survives(self, isolated_env):