PyPI - webget-cli - Versions diffs - 0.11.0__tar.gz → 0.12.1__tar.gz - Mend

webget-cli 0.11.0tar.gz → 0.12.1tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (53) hide show

{webget_cli-0.11.0/webget_cli.egg-info → webget_cli-0.12.1}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.11.0
+Version: 0.12.1
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -23,6 +23,7 @@ Requires-Dist: ddgs
 Requires-Dist: httpx
 Requires-Dist: trafilatura
 Requires-Dist: markdownify
+Requires-Dist: pypdf
 Provides-Extra: browser
 Requires-Dist: crawl4ai>=0.9; extra == "browser"
 Provides-Extra: mcp
@@ -188,13 +189,28 @@ path can reuse them. Secrets are never printed.
       "authenticated": true,
       "state": "success"
     },
+    "metadata": {
+      "author": "Jane Doe",
+      "published_at": "2026-09-01",
+      "site_name": "Campus Portal",
+      "language": "id"
+    },
     "error": null
   }
 }
 ```
+`metadata` (author, published date, site name, language) comes from
+trafilatura extraction on the HTTP path; values are `null` when unknown
+or when the winning strategy was not HTTP.
 Status values: `success | login_required | challenge | blocked | error`.
+The HTTP path also handles non-HTML: JSON becomes a pretty code block,
+CSV becomes a GFM table, RSS/Atom feeds become a link list, PDFs are
+extracted per page (via `pypdf`), plain text passes through. Valid
+non-HTML payloads count as success even when short.
 ## Status detection rules
 | Signal | State |

{webget_cli-0.11.0 → webget_cli-0.12.1}/README.md RENAMED Viewed

@@ -154,13 +154,28 @@ path can reuse them. Secrets are never printed.
       "authenticated": true,
       "state": "success"
     },
+    "metadata": {
+      "author": "Jane Doe",
+      "published_at": "2026-09-01",
+      "site_name": "Campus Portal",
+      "language": "id"
+    },
     "error": null
   }
 }
 ```
+`metadata` (author, published date, site name, language) comes from
+trafilatura extraction on the HTTP path; values are `null` when unknown
+or when the winning strategy was not HTTP.
 Status values: `success | login_required | challenge | blocked | error`.
+The HTTP path also handles non-HTML: JSON becomes a pretty code block,
+CSV becomes a GFM table, RSS/Atom feeds become a link list, PDFs are
+extracted per page (via `pypdf`), plain text passes through. Valid
+non-HTML payloads count as success even when short.
 ## Status detection rules
 | Signal | State |

{webget_cli-0.11.0 → webget_cli-0.12.1}/pyproject.toml RENAMED Viewed

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "webget-cli"
-version = "0.11.0"
+version = "0.12.1"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -26,6 +26,7 @@ dependencies = [
     "httpx",
     "trafilatura",
     "markdownify",
+    "pypdf",
 ]
 [project.urls]

webget_cli-0.12.1/tests/test_base64_strip.py ADDED Viewed

@@ -0,0 +1,55 @@
+"""Ide #3: oversized base64 image payloads are stripped from extracted markdown.
+Bug scope (probed 2026-09-16, see .hermes/plans/2026-09-16_webget-ide03-base64-strip.md):
+the markdownify fallback keeps inline `data:image/...;base64,...` URLs verbatim,
+so a single hero image can carry hundreds of KB into the output. The trafilatura
+path drops images entirely, so only the fallback needs the strip.
+"""
+import webget_cli as webget
+BIG_PAYLOAD = "iVBORw0KGgo" + "A" * 800  # > _BASE64_PAYLOAD_MIN
+SMALL_PAYLOAD = "B" * 120  # <= threshold, must survive
+def _page(payload):
+    return (
+        "<html><head><title>T</title></head><body><article><h1>T</h1>"
+        f'<p><img alt="hero shot" src="data:image/png;base64,{payload}"></p>'
+        "<p>" + ("Body text filler words here to satisfy the length floor. " * 6) + "</p>"
+        "</article></body></html>"
+    )
+def _fallback_only(monkeypatch):
+    """Force the markdownify path (trafilatura returns None)."""
+    import trafilatura
+    monkeypatch.setattr(trafilatura, "extract", lambda *a, **k: None)
+class TestBase64Strip:
+    def test_big_payload_stripped_alt_and_mime_survive(self, monkeypatch):
+        _fallback_only(monkeypatch)
+        text, _ = webget._extract_with_metadata(_page(BIG_PAYLOAD))
+        assert BIG_PAYLOAD not in text
+        assert "data:image/png;base64,stripped" in text
+        assert "hero shot" in text
+    def test_small_payload_preserved(self, monkeypatch):
+        _fallback_only(monkeypatch)
+        text, _ = webget._extract_with_metadata(_page(SMALL_PAYLOAD))
+        assert SMALL_PAYLOAD in text
+    def test_threshold_boundary(self, monkeypatch):
+        _fallback_only(monkeypatch)
+        at_limit = "C" * 200
+        just_over = "D" * 201
+        assert at_limit in webget._extract_with_metadata(_page(at_limit))[0]
+        assert just_over not in webget._extract_with_metadata(_page(just_over))[0]
+    def test_extract_markdown_contract_unchanged(self, monkeypatch):
+        _fallback_only(monkeypatch)
+        out = webget._extract_markdown(_page(BIG_PAYLOAD))
+        assert isinstance(out, str)
+        assert BIG_PAYLOAD not in out

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_mcp_leak_review.py RENAMED Viewed

@@ -24,7 +24,7 @@ LEAK_MARKERS = [
     "/home/",
     "Traceback",
     'File "',
-    "line ",
+    '", line ',  # traceback frames only (plain "line " false-positives on real content, e.g. "online ", "deadline ")
 ]
 # The literal variable NAME may legitimately appear in error messages

webget_cli-0.12.1/tests/test_mcp_metadata.py ADDED Viewed

@@ -0,0 +1,35 @@
+"""Ide #1: metadata passthrough in the MCP tool layer.
+Moved out of tests/test_metadata.py: these import webget_mcp, which needs
+fastmcp/mcp, so they run in the mcp-test CI job (unit job has no fastmcp).
+"""
+class TestMcpMetadataExposure:
+    def test_mcp_fetch_passthrough_has_metadata(self, fresh_cache):
+        import asyncio
+        import webget_mcp
+        res = asyncio.run(
+            webget_mcp.fetch(fresh_cache.url("/long"), strategy="http", no_cache=True)
+        )
+        assert set(res["metadata"]) == {"author", "published_at", "site_name", "language"}
+    def test_mcp_search_fetch_rebuild_has_metadata(self, fresh_cache, monkeypatch):
+        import asyncio
+        import webget_mcp
+        monkeypatch.setattr(
+            webget_mcp.wg,
+            "search",
+            lambda *a, **k: [{"title": "T", "url": fresh_cache.url("/long"), "snippet": "s"}],
+        )
+        out = asyncio.run(webget_mcp.search_fetch("q", n=1, no_cache=True))
+        assert set(out[0]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }

webget_cli-0.12.1/tests/test_metadata.py ADDED Viewed

@@ -0,0 +1,103 @@
+"""Ide #1: metadata from trafilatura exposed in fetch results."""
+import webget_cli as webget
+def _rich_html():
+    return (
+        "<html><head><title>T</title>"
+        '<meta name="author" content="Jane Doe">'
+        '<meta property="article:published_time" content="2026-09-01T10:00:00Z">'
+        "</head><body><article><h1>T</h1><p>"
+        + "Lorem ipsum dolor sit amet consectetur adipiscing elit sed do. " * 10
+        + "</p></article></body></html>"
+    )
+class TestExtractWithMetadata:
+    def test_returns_text_and_metadata(self):
+        text, meta = webget._extract_with_metadata(_rich_html())
+        assert len(text) > 100
+        assert meta["author"] == "Jane Doe"
+        assert "2026-09-01" in (meta.get("published_at") or "")
+    def test_fallback_returns_empty_metadata(self, monkeypatch):
+        import trafilatura
+        monkeypatch.setattr(trafilatura, "extract", lambda *a, **k: None)
+        body = "<p>" + "Fallback body content here yes indeed. " * 5 + "</p>"
+        text, meta = webget._extract_with_metadata("<h1>H</h1>" + body)
+        assert "# H" in text
+        assert meta == {"author": None, "published_at": None, "site_name": None, "language": None}
+    def test_extract_markdown_still_returns_str(self, monkeypatch):
+        # Regression guard: old contract returns plain str, never tuple.
+        import trafilatura
+        monkeypatch.setattr(trafilatura, "extract", lambda *a, **k: None)
+        body = "<p>" + "Fallback body content here yes indeed. " * 5 + "</p>"
+        out = webget._extract_markdown("<h1>H</h1>" + body)
+        assert isinstance(out, str)
+        assert "# H" in out
+class TestFetchHttpMetadata:
+    def test_fetch_http_result_carries_metadata(self, fresh_cache):
+        import asyncio
+        res = asyncio.run(webget.fetch_http(fresh_cache.url("/long"), 6000, timeout=10))
+        assert set(res["metadata"]) == {"author", "published_at", "site_name", "language"}
+    def test_scrape_many_success_carries_metadata(self, fresh_cache):
+        import asyncio
+        out = asyncio.run(webget.scrape_many([fresh_cache.url("/long")], strategy="http"))
+        url = fresh_cache.url("/long")
+        assert out[url]["status"] == "success"
+        assert set(out[url]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }
+    def test_cached_hit_carries_metadata(self, fresh_cache):
+        import asyncio
+        url = fresh_cache.url("/long")
+        asyncio.run(webget.scrape_many([url], strategy="http"))
+        out = asyncio.run(webget.scrape_many([url], strategy="http"))
+        assert out[url]["cached"] is True
+        assert set(out[url]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }
+# NOTE: MCP-layer metadata passthrough tests live in tests/test_mcp_metadata.py
+# (they import webget_mcp, which needs fastmcp; that file runs in the mcp-test CI job).
+class TestOutputExposure:
+    def test_cli_su_json_rebuild_has_metadata(self, fresh_cache, monkeypatch, capsys):
+        import webget_cli as wgcli
+        from webget import cli as cli_mod
+        monkeypatch.setattr(
+            cli_mod,
+            "search",
+            lambda *a, **k: [{"title": "T", "url": fresh_cache.url("/long"), "snippet": "s"}],
+        )
+        monkeypatch.setattr("sys.argv", ["webget", "su", "q", "1", "--json", "--no-cache"])
+        wgcli.main()
+        import json
+        printed = capsys.readouterr().out
+        data = json.loads(printed)
+        url = fresh_cache.url("/long")
+        assert set(data[url]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }

webget_cli-0.12.1/tests/test_nonhtml.py ADDED Viewed

@@ -0,0 +1,110 @@
+"""Ide #2: non-HTML content routing (JSON/text/CSV/feed/PDF to markdown)."""
+import asyncio
+import webget_cli as webget
+class TestJsonRouting:
+    def test_fetch_json_returns_pretty_markdown(self, fresh_cache):
+        url = fresh_cache.url("/json")
+        res = asyncio.run(webget.fetch_http(url, 6000, timeout=10))
+        assert "hello" in res["markdown"]
+        assert "world" in res["markdown"]
+        assert res["metadata"]["site_name"] == "127.0.0.1"
+    def test_scrape_many_json_is_success(self, fresh_cache):
+        url = fresh_cache.url("/json")
+        out = asyncio.run(webget.scrape_many([url], strategy="http", no_cache=True))
+        assert out[url]["status"] == "success"
+        assert out[url]["method"] == "http"
+        assert "hello" in out[url]["markdown"]
+class TestCsvRouting:
+    def test_csv_becomes_gfm_table(self):
+        body = b"name,age\nbudi,25\nsiti,30\n"
+        _title, md, meta = webget._convert_non_html("text/csv", body, "https://example.com/d.csv")
+        assert "| name | age |" in md
+        assert "| budi | 25 |" in md
+        assert meta["site_name"] == "example.com"
+    def test_csv_escapes_pipes(self):
+        body = b"a,b\nx|y,z\n"
+        _, md, _ = webget._convert_non_html("text/csv", body, "https://example.com/d.csv")
+        assert "x\\|y" in md
+class TestFeedRouting:
+    RSS = (
+        b'<?xml version="1.0"?><rss version="2.0"><channel><title>Blog</title>'
+        b"<item><title>Post A</title><link>https://ex.com/a</link>"
+        b"<description>First post here</description></item>"
+        b"<item><title>Post B</title><link>https://ex.com/b</link></item>"
+        b"</channel></rss>"
+    )
+    ATOM = (
+        b'<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom">'
+        b"<title>Blog</title>"
+        b'<entry><title>Entry One</title><link href="https://ex.com/1"/>'
+        b"<summary>Summary one</summary></entry>"
+        b"</feed>"
+    )
+    def test_rss_becomes_link_list(self):
+        _title, md, _meta = webget._convert_non_html(
+            "application/rss+xml", self.RSS, "https://ex.com/feed"
+        )
+        assert "[Post A](https://ex.com/a)" in md
+        assert "[Post B](https://ex.com/b)" in md
+        assert "First post here" in md
+    def test_atom_becomes_link_list(self):
+        _, md, _ = webget._convert_non_html(
+            "application/atom+xml", self.ATOM, "https://ex.com/feed"
+        )
+        assert "[Entry One](https://ex.com/1)" in md
+    def test_xml_without_items_falls_back_to_text(self):
+        _, md, _ = webget._convert_non_html(
+            "application/xml", b"<note><to>u</to></note>", "https://ex.com/n.xml"
+        )
+        assert "to" in md
+class TestPdfRouting:
+    def test_pdf_routing_uses_pypdf(self, monkeypatch):
+        import sys
+        import types
+        fake_page = types.SimpleNamespace(extract_text=lambda: "Hello PDF page one")
+        fake_reader = lambda *a, **k: types.SimpleNamespace(pages=[fake_page])
+        fake_mod = types.SimpleNamespace(PdfReader=fake_reader)
+        monkeypatch.setitem(sys.modules, "pypdf", fake_mod)
+        _title, md, meta = webget._convert_non_html(
+            "application/pdf", b"%PDF-fake", "https://ex.com/d.pdf"
+        )
+        assert "Hello PDF page one" in md
+        assert meta["site_name"] == "ex.com"
+    def test_pdf_missing_dep_errors_clearly(self, monkeypatch):
+        import sys
+        monkeypatch.setitem(sys.modules, "pypdf", None)
+        import pytest
+        with pytest.raises(RuntimeError, match="pypdf"):
+            webget._convert_non_html("application/pdf", b"%PDF-fake", "https://ex.com/d.pdf")
+    def test_pdf_empty_text_errors(self, monkeypatch):
+        import sys
+        import types
+        fake_page = types.SimpleNamespace(extract_text=lambda: "  ")
+        fake_reader = lambda *a, **k: types.SimpleNamespace(pages=[fake_page])
+        fake_mod = types.SimpleNamespace(PdfReader=fake_reader)
+        monkeypatch.setitem(sys.modules, "pypdf", fake_mod)
+        import pytest
+        with pytest.raises(RuntimeError, match="no extractable text"):
+            webget._convert_non_html("application/pdf", b"%PDF-fake", "https://ex.com/d.pdf")

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget/__init__.py RENAMED Viewed

@@ -32,7 +32,14 @@ from .cache import (
 from .cli import main, parse_opts
 from .discovery import discover_urls
 from .firecrawl import fetch_firecrawl, firecrawl_key
-from .http import MAX_RESPONSE_BYTES, ResponseTooLarge, _extract_markdown, fetch_http
+from .http import (
+    MAX_RESPONSE_BYTES,
+    ResponseTooLarge,
+    _convert_non_html,
+    _extract_markdown,
+    _extract_with_metadata,
+    fetch_http,
+)
 from .ladder import (
     _DEFAULT_CONCURRENCY,
     _STRATEGY_MEMORY_TTL,
@@ -97,11 +104,13 @@ __all__ = [
     "_auth_message",
     "_auth_state",
     "_cache_path",
+    "_convert_non_html",
     "_cookie_belongs_to",
     "_crawl4ai_once",
     "_domain_match",
     "_effective_cookies",
     "_extract_markdown",
+    "_extract_with_metadata",
     "_fmt_age",
     "_guard_browser_routes",
     "_hostname_private",

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget/cli.py RENAMED Viewed

@@ -377,6 +377,7 @@ def main():
                     "snippet": r.get("snippet", ""),
                     "scrape_title": got.get("title", ""),
                     "markdown": got.get("markdown", ""),
+                    "metadata": got.get("metadata"),
                     "status": got.get("status", ""),
                     "method": got.get("method", ""),
                     "cached": got.get("cached", False),

webget_cli-0.12.1/webget/http.py ADDED Viewed

@@ -0,0 +1,368 @@
+"""HTTP fast-path fetch + markdown extraction for webget.
+fetch_http() is the cheapest strategy: a plain HTTP GET, manual redirect
+following so the SSRF guard runs on every hop, and a streaming body cap
+so a giant/binary download cannot exhaust memory. Extraction tries
+trafilatura first (clean article text), then markdownify as a fallback.
+"""
+from __future__ import annotations
+import asyncio
+import re
+import time
+import warnings
+from urllib.parse import urlparse
+import httpx
+from .ssrf import SSRFError, _private_ip_for
+# Max response body webget will read from the HTTP fast path (bytes).
+# Guards against memory exhaustion from giant/binary downloads.
+MAX_RESPONSE_BYTES = 25 * 1024 * 1024
+class ResponseTooLarge(Exception):
+    """HTTP response exceeded MAX_RESPONSE_BYTES.
+    Raised by the streaming cap in fetch_http. This is a TERMINAL state
+    for the ladder: retrying a 30MB page in a browser (crawl4ai) would
+    just re-download the same giant body through Chromium, so escalating
+    is pure waste. scrape_many treats it as terminal, not a ladder step.
+    """
+_EMPTY_META = {"author": None, "published_at": None, "site_name": None, "language": None}
+# Inline data:image payloads longer than this (base64 chars) are replaced
+# in extracted markdown; shorter ones are left alone since they cost little
+# and may carry meaningful thumbnails.
+_BASE64_PAYLOAD_MIN = 200
+_DATA_URL_IMAGE_RE = re.compile(r"(data:image/[a-z0-9.+-]+;base64,)([A-Za-z0-9+/=]+)")
+def _strip_base64_images(text):
+    """Replace oversized inline base64 image payloads with 'stripped'.
+    Pages routinely inline images as data URLs; a single hero image can
+    carry hundreds of KB of base64, which the markdownify fallback would
+    otherwise pass through verbatim as token-burning noise. Keeps the
+    mime prefix and the markdown around the URL (alt text lives outside
+    the URL), drops only the payload.
+    """
+    def _repl(m):
+        if len(m.group(2)) <= _BASE64_PAYLOAD_MIN:
+            return m.group(0)
+        return m.group(1) + "stripped"
+    return _DATA_URL_IMAGE_RE.sub(_repl, text)
+def _extract_with_metadata(html):
+    """Extract (text, metadata) from HTML.
+    Tries trafilatura JSON output (clean article text + author/date/site/
+    language metadata) first, then markdownify as a text-only fallback
+    (empty metadata). Returns a (str, dict) tuple; metadata keys are
+    always author/published_at/site_name/language, values None when
+    unknown.
+    """
+    try:
+        import json
+        import trafilatura
+        raw = trafilatura.extract(
+            html,
+            output_format="json",
+            with_metadata=True,
+            include_comments=False,
+            include_tables=True,
+        )
+        if raw:
+            doc = json.loads(raw)
+            text = (doc.get("text") or "").strip()
+            if text and len(text) > 100:
+                return text, {
+                    "author": doc.get("author"),
+                    "published_at": doc.get("date"),
+                    "site_name": doc.get("sitename"),
+                    "language": doc.get("language") or doc.get("lang"),
+                }
+    except Exception:  # noqa: BLE001, S110 - extraction libs vary; fall through
+        pass
+    try:
+        from markdownify import markdownify as md
+        # Feeds/sitemaps served as text/html make BeautifulSoup (via
+        # markdownify) warn per document; the HTML parser still produces
+        # usable markdown, so the warning is noise for a CLI.
+        with warnings.catch_warnings():
+            try:
+                from bs4 import XMLParsedAsHTMLWarning
+                warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
+            except ImportError:
+                pass
+            # bullets="*" and heading_style="ATX" match the previous html2text
+            # output style (verified differential 2026-08-08) so the fallback
+            # stays close to 0.7.2 (semantic parity).
+            converted = md(html, bullets="*", heading_style="ATX").strip()
+            converted = _strip_base64_images(converted)
+        if len(converted) > 50:
+            return converted, dict(_EMPTY_META)
+        return "", dict(_EMPTY_META)
+    except Exception:  # noqa: BLE001 - best-effort extraction, empty is fine
+        return "", dict(_EMPTY_META)
+def _extract_markdown(html):
+    """Try trafilatura (clean article text) then markdownify (full markdown)."""
+    text, _ = _extract_with_metadata(html)
+    return text
+def _convert_non_html(ctype, body, url):
+    """Convert a non-HTML response body to (title, markdown, metadata).
+    Routes by content-type: JSON -> pretty code block, text/* -> plain
+    text, CSV -> GFM table, RSS/Atom/XML feeds -> link list, PDF ->
+    per-page text via pypdf. Raises RuntimeError for unknown types.
+    """
+    from urllib.parse import urlparse
+    host = urlparse(url).hostname or ""
+    meta = {"author": None, "published_at": None, "site_name": host or None, "language": None}
+    low = (ctype or "").lower()
+    if "json" in low:
+        try:
+            import json as _json
+            pretty = _json.dumps(
+                _json.loads(body.decode("utf-8", errors="replace")),
+                indent=2,
+                ensure_ascii=False,
+            )
+        except Exception:  # noqa: BLE001 - malformed JSON falls back to raw text
+            pretty = body.decode("utf-8", errors="replace")
+        return url, f"```json\n{pretty}\n```", meta
+    if low.startswith("text/"):
+        if "csv" in low:
+            return url, _csv_to_gfm(body), meta
+        return url, body.decode("utf-8", errors="replace").strip(), meta
+    if "xml" in low or "rss" in low or "atom" in low or "feed" in low:
+        return url, _feed_to_links(body), meta
+    if "pdf" in low:
+        return url, _pdf_to_text(body), meta
+    raise RuntimeError(f"not HTML ({ctype or 'unknown'})")
+def _pdf_to_text(body):
+    """Extract per-page text from PDF bytes via pypdf."""
+    try:
+        import io as _io
+        from pypdf import PdfReader
+    except ImportError:
+        raise RuntimeError("PDF content requires 'pypdf' (pip install webget-cli)") from None
+    reader = PdfReader(_io.BytesIO(body))
+    parts = []
+    for i, page in enumerate(reader.pages):
+        try:
+            t = page.extract_text() or ""
+        except Exception:  # noqa: BLE001 - one bad page must not kill the doc
+            t = ""
+        if t.strip():
+            parts.append(f"## Page {i + 1}\n\n{t.strip()}")
+    text = "\n\n".join(parts)
+    if not text.strip():
+        raise RuntimeError("PDF has no extractable text")
+    return text
+def _feed_to_links(body):
+    """Convert RSS/Atom XML bytes to a markdown link list."""
+    import xml.etree.ElementTree as _ET
+    try:
+        root = _ET.fromstring(body)
+    except Exception:  # noqa: BLE001 - malformed XML falls back to raw text
+        return body.decode("utf-8", errors="replace").strip()
+    lines = []
+    for item in root.iter("item"):  # RSS
+        title = (item.findtext("title") or "").strip()
+        link = (item.findtext("link") or "").strip()
+        desc = (item.findtext("description") or "").strip()
+        if title and link:
+            lines.append(f"- [{title}]({link})")
+        elif title:
+            lines.append(f"- {title}")
+        if desc:
+            lines.append(f"  > {desc[:200]}")
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+    for entry in root.findall(".//atom:entry", ns):  # Atom
+        t = entry.find("atom:title", ns)
+        title = (t.text or "").strip() if t is not None else ""
+        link = ""
+        for l in entry.findall("atom:link", ns):
+            href = (l.get("href") or "").strip()
+            if href and (l.get("rel", "alternate") == "alternate" or not l.get("rel")):
+                link = href
+                break
+        s = entry.find("atom:summary", ns)
+        desc = (s.text or "").strip() if s is not None else ""
+        if title and link:
+            lines.append(f"- [{title}]({link})")
+        elif title:
+            lines.append(f"- {title}")
+        if desc:
+            lines.append(f"  > {desc[:200]}")
+    if lines:
+        return "\n".join(lines)
+    return body.decode("utf-8", errors="replace").strip()
+def _csv_to_gfm(body):
+    """Convert CSV bytes to a GitHub-flavored markdown table."""
+    import csv as _csv
+    import io as _io
+    try:
+        rows = list(_csv.reader(_io.StringIO(body.decode("utf-8", errors="replace"))))
+    except Exception:  # noqa: BLE001 - malformed CSV falls back to raw text
+        return body.decode("utf-8", errors="replace").strip()
+    rows = [r for r in rows if r]
+    if not rows:
+        return ""
+    esc = lambda c: (c or "").replace("|", "\\|")
+    header = "| " + " | ".join(esc(c) for c in rows[0]) + " |"
+    sep = "| " + " | ".join("---" for _ in rows[0]) + " |"
+    lines = [header, sep]
+    for r in rows[1:]:
+        lines.append("| " + " | ".join(esc(c) for c in r) + " |")
+    return "\n".join(lines)
+async def fetch_http(url, max_chars, cookies=None, headers=None, timeout=15):
+    """Fast path: plain HTTP GET + local markdown extraction.
+    SSRF guard: the initial URL is checked; redirect hops are followed
+    MANUALLY (follow_redirects=False) so every hop is checked against the
+    private-address policy before being requested. Response body is read
+    with a hard cap (MAX_RESPONSE_BYTES).
+    The guard runs the (blocking) resolver in a worker thread bounded by
+    the request timeout: a sick DNS server must not stall the event loop
+    and stretch every httpx timer in a concurrent batch.
+    """
+    # One absolute wall-clock budget for the whole request: DNS guard,
+    # connect, redirects, and body streaming all draw from it.
+    deadline = time.monotonic() + timeout
+    async def ssrf_guard(target):
+        remaining = deadline - time.monotonic()
+        try:
+            # Resolve through the shim so test patches (e.g.
+            # `webget._private_ip_for = guarded`) reach this fast path's
+            # SSRF check; the package module's is the default.
+            try:
+                import webget_cli as _shim
+                _fn = getattr(_shim, "_private_ip_for", None)
+            except ImportError:
+                _fn = None
+            resolver = _fn if (_fn is not None and _fn is not _private_ip_for) else _private_ip_for
+            ip = await asyncio.wait_for(asyncio.to_thread(resolver, target), max(remaining, 0.1))
+        except TimeoutError:
+            raise TimeoutError(f"DNS resolution exceeded {timeout}s") from None
+        if ip is not None:
+            raise SSRFError(f"blocked by SSRF guard: {target} resolves to private address {ip}")
+    await ssrf_guard(url)
+    hdrs = {
+        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/126.0 Safari/537.36"
+    }
+    if headers:
+        hdrs.update(headers)
+    cj = {}
+    if cookies:
+        host = (urlparse(url).hostname or "").lower()
+        now = time.time()
+        if host:
+            for c in cookies:
+                d = (c.get("domain") or "").lstrip(".").lower()
+                if d and (host == d or host.endswith("." + d)):
+                    exp = c.get("expires") or -1
+                    # Skip expired cookies: session cookies (expires<0) and
+                    # future-expiry cookies are sent; past-expiry are not.
+                    if 0 <= exp < now:
+                        continue
+                    cj[c["name"]] = c["value"]
+    current = url
+    redirects = 0
+    async with httpx.AsyncClient(
+        follow_redirects=False, timeout=timeout, headers=hdrs, cookies=cj
+    ) as client:
+        while True:
+            await ssrf_guard(current)
+            # stream=True is REQUIRED: client.get() would buffer the whole
+            # body into memory before our cap could stop it.
+            async with client.stream("GET", current) as r:
+                if r.status_code in (301, 302, 303, 307, 308):
+                    loc = r.headers.get("location")
+                    if not loc:
+                        break
+                    redirects += 1
+                    if redirects > 20:
+                        raise RuntimeError("too many redirects")
+                    current = str(httpx.URL(current).join(loc))
+                    continue
+                ctype = r.headers.get("content-type", "")
+                # Read with a hard cap while streaming, so a giant/binary
+                # body cannot exhaust memory. httpx's timeout bounds a
+                # single socket operation only, so a server that slow-drips
+                # the body in small chunks over minutes can keep it alive
+                # far past the deadline; enforce an absolute wall-clock cap
+                # here so one slow URL cannot stall the whole batch.
+                chunks = []
+                total = 0
+                async for chunk in r.aiter_bytes():
+                    if time.monotonic() > deadline:
+                        raise TimeoutError(f"streaming body exceeded {timeout}s deadline")
+                    total += len(chunk)
+                    if total > MAX_RESPONSE_BYTES:
+                        raise ResponseTooLarge(f"response too large (> {MAX_RESPONSE_BYTES} bytes)")
+                    chunks.append(chunk)
+                raw_body = b"".join(chunks)
+                low_ctype = ctype.lower()
+                if "html" in low_ctype or (
+                    "text" in low_ctype and "csv" not in low_ctype and "xml" not in low_ctype
+                ):
+                    html = raw_body.decode("utf-8", errors="replace")
+                else:
+                    title, md, meta = _convert_non_html(ctype, raw_body, current)
+                    return {
+                        "title": title,
+                        "markdown": md[:max_chars],
+                        "metadata": meta,
+                        "non_html": True,
+                        "status_code": r.status_code,
+                        "html": "",
+                    }
+                title = ""
+                m = re.search(r"<title[^>]*>(.*?)</title>", html, re.DOTALL | re.IGNORECASE)
+                if m:
+                    title = re.sub(r"\s+", " ", m.group(1)).strip()
+                md, meta = await asyncio.to_thread(_extract_with_metadata, html)
+                return {
+                    "title": title,
+                    "markdown": md[:max_chars],
+                    "metadata": meta,
+                    "status_code": r.status_code,
+                    "html": html[:8000],
+                }

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget/ladder.py RENAMED Viewed

@@ -144,10 +144,16 @@ def _reorder_steps_by_domain(steps, url):
     return [preferred] + [s for s in steps if s != preferred]
+def _empty_meta():
+    """Fresh empty metadata dict (never shared mutable module state)."""
+    return {"author": None, "published_at": None, "site_name": None, "language": None}
 def _normalize_hit(hit):
     return {
         "title": hit.get("title", ""),
         "markdown": hit.get("markdown", ""),
+        "metadata": hit.get("metadata") or _empty_meta(),
         "status": "success",
         "method": "cache",
         "cached": True,
@@ -370,6 +376,7 @@ async def scrape_many(
             results[u] = {
                 "title": "",
                 "markdown": "",
+                "metadata": _empty_meta(),
                 "status": "error",
                 "method": "",
                 "cached": False,
@@ -406,7 +413,11 @@ async def scrape_many(
             reasons[url].append(("error", method, detail))
             return None
         state, authenticated = _auth_state(res, profile)
-        if state == "success" and len((res.get("markdown") or "").strip()) >= 100:
+        # Thin-check guards HTML extraction (an empty/login page misread as
+        # content). Non-HTML conversions (JSON/CSV/feed/PDF) are already
+        # validated by their parser, so a short-but-valid payload counts.
+        thin_ok = len((res.get("markdown") or "").strip()) >= 100 or res.get("non_html")
+        if state == "success" and thin_ok:
             auth = {"profile": profile, "authenticated": authenticated, "state": state}
             # Record which strategy won for this domain so future 'auto'
             # batches can try it first (per-domain strategy memory).
@@ -416,6 +427,7 @@ async def scrape_many(
             out = {
                 "title": res.get("title", ""),
                 "markdown": res.get("markdown", ""),
+                "metadata": res.get("metadata") or _empty_meta(),
                 "status": "success",
                 "method": method,
                 "cached": False,
@@ -460,6 +472,7 @@ async def scrape_many(
                     return url, {
                         "title": "",
                         "markdown": "",
+                        "metadata": _empty_meta(),
                         "status": "error",
                         "method": "http",
                         "cached": False,
@@ -602,6 +615,7 @@ async def scrape_many(
         results[url] = {
             "title": "",
             "markdown": "",
+            "metadata": _empty_meta(),
             "status": state,
             "method": method,
             "cached": False,

{webget_cli-0.11.0 → webget_cli-0.12.1/webget_cli.egg-info}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.11.0
+Version: 0.12.1
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -23,6 +23,7 @@ Requires-Dist: ddgs
 Requires-Dist: httpx
 Requires-Dist: trafilatura
 Requires-Dist: markdownify
+Requires-Dist: pypdf
 Provides-Extra: browser
 Requires-Dist: crawl4ai>=0.9; extra == "browser"
 Provides-Extra: mcp
@@ -188,13 +189,28 @@ path can reuse them. Secrets are never printed.
       "authenticated": true,
       "state": "success"
     },
+    "metadata": {
+      "author": "Jane Doe",
+      "published_at": "2026-09-01",
+      "site_name": "Campus Portal",
+      "language": "id"
+    },
     "error": null
   }
 }
 ```
+`metadata` (author, published date, site name, language) comes from
+trafilatura extraction on the HTTP path; values are `null` when unknown
+or when the winning strategy was not HTTP.
 Status values: `success | login_required | challenge | blocked | error`.
+The HTTP path also handles non-HTML: JSON becomes a pretty code block,
+CSV becomes a GFM table, RSS/Atom feeds become a link list, PDFs are
+extracted per page (via `pypdf`), plain text passes through. Valid
+non-HTML payloads count as success even when short.
 ## Status detection rules
 | Signal | State |

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget_cli.egg-info/SOURCES.txt RENAMED Viewed

@@ -10,6 +10,7 @@ tests/test_adversarial_http.py
 tests/test_adversarial_mcp.py
 tests/test_adversarial_ssrf.py
 tests/test_auth_review.py
+tests/test_base64_strip.py
 tests/test_browser_ssrf.py
 tests/test_cache_review.py
 tests/test_concurrency_review.py
@@ -21,9 +22,12 @@ tests/test_ladder_retry.py
 tests/test_login_flow.py
 tests/test_mcp_leak_review.py
 tests/test_mcp_map.py
+tests/test_mcp_metadata.py
 tests/test_mcp_profile.py
 tests/test_mcp_server.py
 tests/test_mcp_smoke.py
+tests/test_metadata.py
+tests/test_nonhtml.py
 tests/test_security_review.py
 tests/test_size_review.py
 tests/test_ssrf_dual_dns.py

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget_cli.egg-info/requires.txt RENAMED Viewed

@@ -2,6 +2,7 @@ ddgs
 httpx
 trafilatura
 markdownify
+pypdf
 [browser]
 crawl4ai>=0.9

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget_mcp.py RENAMED Viewed

@@ -230,6 +230,7 @@ async def search_fetch(
                 "snippet": r.get("snippet", ""),
                 "scrape_title": got.get("title", ""),
                 "markdown": got.get("markdown", ""),
+                "metadata": got.get("metadata"),
                 "status": got.get("status", ""),
                 "method": got.get("method", ""),
                 "cached": got.get("cached", False),

webget_cli-0.11.0/webget/http.py DELETED Viewed

@@ -1,174 +0,0 @@
-"""HTTP fast-path fetch + markdown extraction for webget.
-fetch_http() is the cheapest strategy: a plain HTTP GET, manual redirect
-following so the SSRF guard runs on every hop, and a streaming body cap
-so a giant/binary download cannot exhaust memory. Extraction tries
-trafilatura first (clean article text), then markdownify as a fallback.
-"""
-from __future__ import annotations
-import asyncio
-import re
-import time
-import warnings
-from urllib.parse import urlparse
-import httpx
-from .ssrf import SSRFError, _private_ip_for
-# Max response body webget will read from the HTTP fast path (bytes).
-# Guards against memory exhaustion from giant/binary downloads.
-MAX_RESPONSE_BYTES = 25 * 1024 * 1024
-class ResponseTooLarge(Exception):
-    """HTTP response exceeded MAX_RESPONSE_BYTES.
-    Raised by the streaming cap in fetch_http. This is a TERMINAL state
-    for the ladder: retrying a 30MB page in a browser (crawl4ai) would
-    just re-download the same giant body through Chromium, so escalating
-    is pure waste. scrape_many treats it as terminal, not a ladder step.
-    """
-def _extract_markdown(html):
-    """Try trafilatura (clean article text) then markdownify (full markdown)."""
-    try:
-        import trafilatura
-        text = trafilatura.extract(html, include_comments=False, include_tables=True)
-        if text and len(text.strip()) > 100:
-            return text.strip()
-    except Exception:  # noqa: BLE001, S110 - extraction libs vary; fall through
-        pass
-    try:
-        from markdownify import markdownify as md
-        # Feeds/sitemaps served as text/html make BeautifulSoup (via
-        # markdownify) warn per document; the HTML parser still produces
-        # usable markdown, so the warning is noise for a CLI.
-        with warnings.catch_warnings():
-            try:
-                from bs4 import XMLParsedAsHTMLWarning
-                warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
-            except ImportError:
-                pass
-            # bullets="*" and heading_style="ATX" match the previous html2text
-            # output style (verified differential 2026-08-08) so the fallback
-            # stays close to 0.7.2 (semantic parity).
-            converted = md(html, bullets="*", heading_style="ATX").strip()
-        return converted if len(converted) > 50 else ""
-    except Exception:  # noqa: BLE001 - best-effort extraction, empty is fine
-        return ""
-async def fetch_http(url, max_chars, cookies=None, headers=None, timeout=15):
-    """Fast path: plain HTTP GET + local markdown extraction.
-    SSRF guard: the initial URL is checked; redirect hops are followed
-    MANUALLY (follow_redirects=False) so every hop is checked against the
-    private-address policy before being requested. Response body is read
-    with a hard cap (MAX_RESPONSE_BYTES).
-    The guard runs the (blocking) resolver in a worker thread bounded by
-    the request timeout: a sick DNS server must not stall the event loop
-    and stretch every httpx timer in a concurrent batch.
-    """
-    # One absolute wall-clock budget for the whole request: DNS guard,
-    # connect, redirects, and body streaming all draw from it.
-    deadline = time.monotonic() + timeout
-    async def ssrf_guard(target):
-        remaining = deadline - time.monotonic()
-        try:
-            # Resolve through the shim so test patches (e.g.
-            # `webget._private_ip_for = guarded`) reach this fast path's
-            # SSRF check; the package module's is the default.
-            try:
-                import webget_cli as _shim
-                _fn = getattr(_shim, "_private_ip_for", None)
-            except ImportError:
-                _fn = None
-            resolver = _fn if (_fn is not None and _fn is not _private_ip_for) else _private_ip_for
-            ip = await asyncio.wait_for(asyncio.to_thread(resolver, target), max(remaining, 0.1))
-        except TimeoutError:
-            raise TimeoutError(f"DNS resolution exceeded {timeout}s") from None
-        if ip is not None:
-            raise SSRFError(f"blocked by SSRF guard: {target} resolves to private address {ip}")
-    await ssrf_guard(url)
-    hdrs = {
-        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-        "(KHTML, like Gecko) Chrome/126.0 Safari/537.36"
-    }
-    if headers:
-        hdrs.update(headers)
-    cj = {}
-    if cookies:
-        host = (urlparse(url).hostname or "").lower()
-        now = time.time()
-        if host:
-            for c in cookies:
-                d = (c.get("domain") or "").lstrip(".").lower()
-                if d and (host == d or host.endswith("." + d)):
-                    exp = c.get("expires") or -1
-                    # Skip expired cookies: session cookies (expires<0) and
-                    # future-expiry cookies are sent; past-expiry are not.
-                    if 0 <= exp < now:
-                        continue
-                    cj[c["name"]] = c["value"]
-    current = url
-    redirects = 0
-    async with httpx.AsyncClient(
-        follow_redirects=False, timeout=timeout, headers=hdrs, cookies=cj
-    ) as client:
-        while True:
-            await ssrf_guard(current)
-            # stream=True is REQUIRED: client.get() would buffer the whole
-            # body into memory before our cap could stop it.
-            async with client.stream("GET", current) as r:
-                if r.status_code in (301, 302, 303, 307, 308):
-                    loc = r.headers.get("location")
-                    if not loc:
-                        break
-                    redirects += 1
-                    if redirects > 20:
-                        raise RuntimeError("too many redirects")
-                    current = str(httpx.URL(current).join(loc))
-                    continue
-                ctype = r.headers.get("content-type", "")
-                if "html" not in ctype and "text" not in ctype:
-                    raise RuntimeError(f"not HTML ({ctype or 'unknown'})")
-                # Read with a hard cap while streaming, so a giant/binary
-                # body cannot exhaust memory. httpx's timeout bounds a
-                # single socket operation only, so a server that slow-drips
-                # the body in small chunks over minutes can keep it alive
-                # far past the deadline; enforce an absolute wall-clock cap
-                # here so one slow URL cannot stall the whole batch.
-                chunks = []
-                total = 0
-                async for chunk in r.aiter_bytes():
-                    if time.monotonic() > deadline:
-                        raise TimeoutError(f"streaming body exceeded {timeout}s deadline")
-                    total += len(chunk)
-                    if total > MAX_RESPONSE_BYTES:
-                        raise ResponseTooLarge(f"response too large (> {MAX_RESPONSE_BYTES} bytes)")
-                    chunks.append(chunk)
-                html = b"".join(chunks).decode("utf-8", errors="replace")
-                title = ""
-                m = re.search(r"<title[^>]*>(.*?)</title>", html, re.DOTALL | re.IGNORECASE)
-                if m:
-                    title = re.sub(r"\s+", " ", m.group(1)).strip()
-                md = await asyncio.to_thread(_extract_markdown, html)
-                return {
-                    "title": title,
-                    "markdown": md[:max_chars],
-                    "status_code": r.status_code,
-                    "html": html[:8000],
-                }

{webget_cli-0.11.0 → webget_cli-0.12.1}/LICENSE RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/setup.cfg RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_adversarial_auth.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_adversarial_cache.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_adversarial_concurrency.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_adversarial_http.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_adversarial_mcp.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_adversarial_ssrf.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_auth_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_browser_ssrf.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_cache_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_concurrency_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_discovery_map.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_extraction_markdown.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_firecrawl_policy.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_integration_ladder.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_ladder_retry.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_login_flow.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_mcp_map.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_mcp_profile.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_mcp_server.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_mcp_smoke.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_security_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_size_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_ssrf_dual_dns.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/tests/test_webget.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget/cache.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget/discovery.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget/firecrawl.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget/profile.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget/search.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget/ssrf.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget_cli.egg-info/dependency_links.txt RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget_cli.egg-info/entry_points.txt RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget_cli.egg-info/top_level.txt RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.1}/webget_cli.py RENAMED Viewed

File without changes

webget-cli 0.11.0__tar.gz → 0.12.1__tar.gz

webget-cli 0.11.0tar.gz → 0.12.1tar.gz