PyPI - webget-cli - Versions diffs - 0.11.0__tar.gz → 0.12.0__tar.gz - Mend

webget-cli 0.11.0tar.gz → 0.12.0tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (50) hide show

{webget_cli-0.11.0/webget_cli.egg-info → webget_cli-0.12.0}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.11.0
+Version: 0.12.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -23,6 +23,7 @@ Requires-Dist: ddgs
 Requires-Dist: httpx
 Requires-Dist: trafilatura
 Requires-Dist: markdownify
+Requires-Dist: pypdf
 Provides-Extra: browser
 Requires-Dist: crawl4ai>=0.9; extra == "browser"
 Provides-Extra: mcp
@@ -188,13 +189,28 @@ path can reuse them. Secrets are never printed.
       "authenticated": true,
       "state": "success"
     },
+    "metadata": {
+      "author": "Jane Doe",
+      "published_at": "2026-09-01",
+      "site_name": "Campus Portal",
+      "language": "id"
+    },
     "error": null
   }
 }
 ```
+`metadata` (author, published date, site name, language) comes from
+trafilatura extraction on the HTTP path; values are `null` when unknown
+or when the winning strategy was not HTTP.
 Status values: `success | login_required | challenge | blocked | error`.
+The HTTP path also handles non-HTML: JSON becomes a pretty code block,
+CSV becomes a GFM table, RSS/Atom feeds become a link list, PDFs are
+extracted per page (via `pypdf`), plain text passes through. Valid
+non-HTML payloads count as success even when short.
 ## Status detection rules
 | Signal | State |

{webget_cli-0.11.0 → webget_cli-0.12.0}/README.md RENAMED Viewed

@@ -154,13 +154,28 @@ path can reuse them. Secrets are never printed.
       "authenticated": true,
       "state": "success"
     },
+    "metadata": {
+      "author": "Jane Doe",
+      "published_at": "2026-09-01",
+      "site_name": "Campus Portal",
+      "language": "id"
+    },
     "error": null
   }
 }
 ```
+`metadata` (author, published date, site name, language) comes from
+trafilatura extraction on the HTTP path; values are `null` when unknown
+or when the winning strategy was not HTTP.
 Status values: `success | login_required | challenge | blocked | error`.
+The HTTP path also handles non-HTML: JSON becomes a pretty code block,
+CSV becomes a GFM table, RSS/Atom feeds become a link list, PDFs are
+extracted per page (via `pypdf`), plain text passes through. Valid
+non-HTML payloads count as success even when short.
 ## Status detection rules
 | Signal | State |

{webget_cli-0.11.0 → webget_cli-0.12.0}/pyproject.toml RENAMED Viewed

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "webget-cli"
-version = "0.11.0"
+version = "0.12.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -26,6 +26,7 @@ dependencies = [
     "httpx",
     "trafilatura",
     "markdownify",
+    "pypdf",
 ]
 [project.urls]

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_mcp_leak_review.py RENAMED Viewed

@@ -24,7 +24,7 @@ LEAK_MARKERS = [
     "/home/",
     "Traceback",
     'File "',
-    "line ",
+    '", line ',  # traceback frames only (plain "line " false-positives on real content, e.g. "online ", "deadline ")
 ]
 # The literal variable NAME may legitimately appear in error messages

webget_cli-0.12.0/tests/test_metadata.py ADDED Viewed

@@ -0,0 +1,129 @@
+"""Ide #1: metadata from trafilatura exposed in fetch results."""
+import webget_cli as webget
+def _rich_html():
+    return (
+        "<html><head><title>T</title>"
+        '<meta name="author" content="Jane Doe">'
+        '<meta property="article:published_time" content="2026-09-01T10:00:00Z">'
+        "</head><body><article><h1>T</h1><p>"
+        + "Lorem ipsum dolor sit amet consectetur adipiscing elit sed do. " * 10
+        + "</p></article></body></html>"
+    )
+class TestExtractWithMetadata:
+    def test_returns_text_and_metadata(self):
+        text, meta = webget._extract_with_metadata(_rich_html())
+        assert len(text) > 100
+        assert meta["author"] == "Jane Doe"
+        assert "2026-09-01" in (meta.get("published_at") or "")
+    def test_fallback_returns_empty_metadata(self, monkeypatch):
+        import trafilatura
+        monkeypatch.setattr(trafilatura, "extract", lambda *a, **k: None)
+        body = "<p>" + "Fallback body content here yes indeed. " * 5 + "</p>"
+        text, meta = webget._extract_with_metadata("<h1>H</h1>" + body)
+        assert "# H" in text
+        assert meta == {"author": None, "published_at": None, "site_name": None, "language": None}
+    def test_extract_markdown_still_returns_str(self, monkeypatch):
+        # Regression guard: old contract returns plain str, never tuple.
+        import trafilatura
+        monkeypatch.setattr(trafilatura, "extract", lambda *a, **k: None)
+        body = "<p>" + "Fallback body content here yes indeed. " * 5 + "</p>"
+        out = webget._extract_markdown("<h1>H</h1>" + body)
+        assert isinstance(out, str)
+        assert "# H" in out
+class TestFetchHttpMetadata:
+    def test_fetch_http_result_carries_metadata(self, fresh_cache):
+        import asyncio
+        res = asyncio.run(webget.fetch_http(fresh_cache.url("/long"), 6000, timeout=10))
+        assert set(res["metadata"]) == {"author", "published_at", "site_name", "language"}
+    def test_scrape_many_success_carries_metadata(self, fresh_cache):
+        import asyncio
+        out = asyncio.run(webget.scrape_many([fresh_cache.url("/long")], strategy="http"))
+        url = fresh_cache.url("/long")
+        assert out[url]["status"] == "success"
+        assert set(out[url]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }
+    def test_cached_hit_carries_metadata(self, fresh_cache):
+        import asyncio
+        url = fresh_cache.url("/long")
+        asyncio.run(webget.scrape_many([url], strategy="http"))
+        out = asyncio.run(webget.scrape_many([url], strategy="http"))
+        assert out[url]["cached"] is True
+        assert set(out[url]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }
+class TestOutputExposure:
+    def test_mcp_fetch_passthrough_has_metadata(self, fresh_cache):
+        import asyncio
+        import webget_mcp
+        res = asyncio.run(
+            webget_mcp.fetch(fresh_cache.url("/long"), strategy="http", no_cache=True)
+        )
+        assert set(res["metadata"]) == {"author", "published_at", "site_name", "language"}
+    def test_mcp_search_fetch_rebuild_has_metadata(self, fresh_cache, monkeypatch):
+        import asyncio
+        import webget_mcp
+        monkeypatch.setattr(
+            webget_mcp.wg,
+            "search",
+            lambda *a, **k: [{"title": "T", "url": fresh_cache.url("/long"), "snippet": "s"}],
+        )
+        out = asyncio.run(webget_mcp.search_fetch("q", n=1, no_cache=True))
+        assert set(out[0]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }
+    def test_cli_su_json_rebuild_has_metadata(self, fresh_cache, monkeypatch, capsys):
+        import webget_cli as wgcli
+        from webget import cli as cli_mod
+        monkeypatch.setattr(
+            cli_mod,
+            "search",
+            lambda *a, **k: [{"title": "T", "url": fresh_cache.url("/long"), "snippet": "s"}],
+        )
+        monkeypatch.setattr("sys.argv", ["webget", "su", "q", "1", "--json", "--no-cache"])
+        wgcli.main()
+        import json
+        printed = capsys.readouterr().out
+        data = json.loads(printed)
+        url = fresh_cache.url("/long")
+        assert set(data[url]["metadata"]) == {
+            "author",
+            "published_at",
+            "site_name",
+            "language",
+        }

webget_cli-0.12.0/tests/test_nonhtml.py ADDED Viewed

@@ -0,0 +1,110 @@
+"""Ide #2: non-HTML content routing (JSON/text/CSV/feed/PDF to markdown)."""
+import asyncio
+import webget_cli as webget
+class TestJsonRouting:
+    def test_fetch_json_returns_pretty_markdown(self, fresh_cache):
+        url = fresh_cache.url("/json")
+        res = asyncio.run(webget.fetch_http(url, 6000, timeout=10))
+        assert "hello" in res["markdown"]
+        assert "world" in res["markdown"]
+        assert res["metadata"]["site_name"] == "127.0.0.1"
+    def test_scrape_many_json_is_success(self, fresh_cache):
+        url = fresh_cache.url("/json")
+        out = asyncio.run(webget.scrape_many([url], strategy="http", no_cache=True))
+        assert out[url]["status"] == "success"
+        assert out[url]["method"] == "http"
+        assert "hello" in out[url]["markdown"]
+class TestCsvRouting:
+    def test_csv_becomes_gfm_table(self):
+        body = b"name,age\nbudi,25\nsiti,30\n"
+        _title, md, meta = webget._convert_non_html("text/csv", body, "https://example.com/d.csv")
+        assert "| name | age |" in md
+        assert "| budi | 25 |" in md
+        assert meta["site_name"] == "example.com"
+    def test_csv_escapes_pipes(self):
+        body = b"a,b\nx|y,z\n"
+        _, md, _ = webget._convert_non_html("text/csv", body, "https://example.com/d.csv")
+        assert "x\\|y" in md
+class TestFeedRouting:
+    RSS = (
+        b'<?xml version="1.0"?><rss version="2.0"><channel><title>Blog</title>'
+        b"<item><title>Post A</title><link>https://ex.com/a</link>"
+        b"<description>First post here</description></item>"
+        b"<item><title>Post B</title><link>https://ex.com/b</link></item>"
+        b"</channel></rss>"
+    )
+    ATOM = (
+        b'<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom">'
+        b"<title>Blog</title>"
+        b'<entry><title>Entry One</title><link href="https://ex.com/1"/>'
+        b"<summary>Summary one</summary></entry>"
+        b"</feed>"
+    )
+    def test_rss_becomes_link_list(self):
+        _title, md, _meta = webget._convert_non_html(
+            "application/rss+xml", self.RSS, "https://ex.com/feed"
+        )
+        assert "[Post A](https://ex.com/a)" in md
+        assert "[Post B](https://ex.com/b)" in md
+        assert "First post here" in md
+    def test_atom_becomes_link_list(self):
+        _, md, _ = webget._convert_non_html(
+            "application/atom+xml", self.ATOM, "https://ex.com/feed"
+        )
+        assert "[Entry One](https://ex.com/1)" in md
+    def test_xml_without_items_falls_back_to_text(self):
+        _, md, _ = webget._convert_non_html(
+            "application/xml", b"<note><to>u</to></note>", "https://ex.com/n.xml"
+        )
+        assert "to" in md
+class TestPdfRouting:
+    def test_pdf_routing_uses_pypdf(self, monkeypatch):
+        import sys
+        import types
+        fake_page = types.SimpleNamespace(extract_text=lambda: "Hello PDF page one")
+        fake_reader = lambda *a, **k: types.SimpleNamespace(pages=[fake_page])
+        fake_mod = types.SimpleNamespace(PdfReader=fake_reader)
+        monkeypatch.setitem(sys.modules, "pypdf", fake_mod)
+        _title, md, meta = webget._convert_non_html(
+            "application/pdf", b"%PDF-fake", "https://ex.com/d.pdf"
+        )
+        assert "Hello PDF page one" in md
+        assert meta["site_name"] == "ex.com"
+    def test_pdf_missing_dep_errors_clearly(self, monkeypatch):
+        import sys
+        monkeypatch.setitem(sys.modules, "pypdf", None)
+        import pytest
+        with pytest.raises(RuntimeError, match="pypdf"):
+            webget._convert_non_html("application/pdf", b"%PDF-fake", "https://ex.com/d.pdf")
+    def test_pdf_empty_text_errors(self, monkeypatch):
+        import sys
+        import types
+        fake_page = types.SimpleNamespace(extract_text=lambda: "  ")
+        fake_reader = lambda *a, **k: types.SimpleNamespace(pages=[fake_page])
+        fake_mod = types.SimpleNamespace(PdfReader=fake_reader)
+        monkeypatch.setitem(sys.modules, "pypdf", fake_mod)
+        import pytest
+        with pytest.raises(RuntimeError, match="no extractable text"):
+            webget._convert_non_html("application/pdf", b"%PDF-fake", "https://ex.com/d.pdf")

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/__init__.py RENAMED Viewed

@@ -32,7 +32,14 @@ from .cache import (
 from .cli import main, parse_opts
 from .discovery import discover_urls
 from .firecrawl import fetch_firecrawl, firecrawl_key
-from .http import MAX_RESPONSE_BYTES, ResponseTooLarge, _extract_markdown, fetch_http
+from .http import (
+    MAX_RESPONSE_BYTES,
+    ResponseTooLarge,
+    _convert_non_html,
+    _extract_markdown,
+    _extract_with_metadata,
+    fetch_http,
+)
 from .ladder import (
     _DEFAULT_CONCURRENCY,
     _STRATEGY_MEMORY_TTL,
@@ -97,11 +104,13 @@ __all__ = [
     "_auth_message",
     "_auth_state",
     "_cache_path",
+    "_convert_non_html",
     "_cookie_belongs_to",
     "_crawl4ai_once",
     "_domain_match",
     "_effective_cookies",
     "_extract_markdown",
+    "_extract_with_metadata",
     "_fmt_age",
     "_guard_browser_routes",
     "_hostname_private",

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/cli.py RENAMED Viewed

@@ -377,6 +377,7 @@ def main():
                     "snippet": r.get("snippet", ""),
                     "scrape_title": got.get("title", ""),
                     "markdown": got.get("markdown", ""),
+                    "metadata": got.get("metadata"),
                     "status": got.get("status", ""),
                     "method": got.get("method", ""),
                     "cached": got.get("cached", False),

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/http.py RENAMED Viewed

@@ -33,14 +33,40 @@ class ResponseTooLarge(Exception):
     """
-def _extract_markdown(html):
-    """Try trafilatura (clean article text) then markdownify (full markdown)."""
+_EMPTY_META = {"author": None, "published_at": None, "site_name": None, "language": None}
+def _extract_with_metadata(html):
+    """Extract (text, metadata) from HTML.
+    Tries trafilatura JSON output (clean article text + author/date/site/
+    language metadata) first, then markdownify as a text-only fallback
+    (empty metadata). Returns a (str, dict) tuple; metadata keys are
+    always author/published_at/site_name/language, values None when
+    unknown.
+    """
     try:
+        import json
         import trafilatura
-        text = trafilatura.extract(html, include_comments=False, include_tables=True)
-        if text and len(text.strip()) > 100:
-            return text.strip()
+        raw = trafilatura.extract(
+            html,
+            output_format="json",
+            with_metadata=True,
+            include_comments=False,
+            include_tables=True,
+        )
+        if raw:
+            doc = json.loads(raw)
+            text = (doc.get("text") or "").strip()
+            if text and len(text) > 100:
+                return text, {
+                    "author": doc.get("author"),
+                    "published_at": doc.get("date"),
+                    "site_name": doc.get("sitename"),
+                    "language": doc.get("language") or doc.get("lang"),
+                }
     except Exception:  # noqa: BLE001, S110 - extraction libs vary; fall through
         pass
     try:
@@ -60,9 +86,138 @@ def _extract_markdown(html):
             # output style (verified differential 2026-08-08) so the fallback
             # stays close to 0.7.2 (semantic parity).
             converted = md(html, bullets="*", heading_style="ATX").strip()
-        return converted if len(converted) > 50 else ""
+        if len(converted) > 50:
+            return converted, dict(_EMPTY_META)
+        return "", dict(_EMPTY_META)
     except Exception:  # noqa: BLE001 - best-effort extraction, empty is fine
+        return "", dict(_EMPTY_META)
+def _extract_markdown(html):
+    """Try trafilatura (clean article text) then markdownify (full markdown)."""
+    text, _ = _extract_with_metadata(html)
+    return text
+def _convert_non_html(ctype, body, url):
+    """Convert a non-HTML response body to (title, markdown, metadata).
+    Routes by content-type: JSON -> pretty code block, text/* -> plain
+    text, CSV -> GFM table, RSS/Atom/XML feeds -> link list, PDF ->
+    per-page text via pypdf. Raises RuntimeError for unknown types.
+    """
+    from urllib.parse import urlparse
+    host = urlparse(url).hostname or ""
+    meta = {"author": None, "published_at": None, "site_name": host or None, "language": None}
+    low = (ctype or "").lower()
+    if "json" in low:
+        try:
+            import json as _json
+            pretty = _json.dumps(
+                _json.loads(body.decode("utf-8", errors="replace")),
+                indent=2,
+                ensure_ascii=False,
+            )
+        except Exception:  # noqa: BLE001 - malformed JSON falls back to raw text
+            pretty = body.decode("utf-8", errors="replace")
+        return url, f"```json\n{pretty}\n```", meta
+    if low.startswith("text/"):
+        if "csv" in low:
+            return url, _csv_to_gfm(body), meta
+        return url, body.decode("utf-8", errors="replace").strip(), meta
+    if "xml" in low or "rss" in low or "atom" in low or "feed" in low:
+        return url, _feed_to_links(body), meta
+    if "pdf" in low:
+        return url, _pdf_to_text(body), meta
+    raise RuntimeError(f"not HTML ({ctype or 'unknown'})")
+def _pdf_to_text(body):
+    """Extract per-page text from PDF bytes via pypdf."""
+    try:
+        import io as _io
+        from pypdf import PdfReader
+    except ImportError:
+        raise RuntimeError("PDF content requires 'pypdf' (pip install webget-cli)") from None
+    reader = PdfReader(_io.BytesIO(body))
+    parts = []
+    for i, page in enumerate(reader.pages):
+        try:
+            t = page.extract_text() or ""
+        except Exception:  # noqa: BLE001 - one bad page must not kill the doc
+            t = ""
+        if t.strip():
+            parts.append(f"## Page {i + 1}\n\n{t.strip()}")
+    text = "\n\n".join(parts)
+    if not text.strip():
+        raise RuntimeError("PDF has no extractable text")
+    return text
+def _feed_to_links(body):
+    """Convert RSS/Atom XML bytes to a markdown link list."""
+    import xml.etree.ElementTree as _ET
+    try:
+        root = _ET.fromstring(body)
+    except Exception:  # noqa: BLE001 - malformed XML falls back to raw text
+        return body.decode("utf-8", errors="replace").strip()
+    lines = []
+    for item in root.iter("item"):  # RSS
+        title = (item.findtext("title") or "").strip()
+        link = (item.findtext("link") or "").strip()
+        desc = (item.findtext("description") or "").strip()
+        if title and link:
+            lines.append(f"- [{title}]({link})")
+        elif title:
+            lines.append(f"- {title}")
+        if desc:
+            lines.append(f"  > {desc[:200]}")
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+    for entry in root.findall(".//atom:entry", ns):  # Atom
+        t = entry.find("atom:title", ns)
+        title = (t.text or "").strip() if t is not None else ""
+        link = ""
+        for l in entry.findall("atom:link", ns):
+            href = (l.get("href") or "").strip()
+            if href and (l.get("rel", "alternate") == "alternate" or not l.get("rel")):
+                link = href
+                break
+        s = entry.find("atom:summary", ns)
+        desc = (s.text or "").strip() if s is not None else ""
+        if title and link:
+            lines.append(f"- [{title}]({link})")
+        elif title:
+            lines.append(f"- {title}")
+        if desc:
+            lines.append(f"  > {desc[:200]}")
+    if lines:
+        return "\n".join(lines)
+    return body.decode("utf-8", errors="replace").strip()
+def _csv_to_gfm(body):
+    """Convert CSV bytes to a GitHub-flavored markdown table."""
+    import csv as _csv
+    import io as _io
+    try:
+        rows = list(_csv.reader(_io.StringIO(body.decode("utf-8", errors="replace"))))
+    except Exception:  # noqa: BLE001 - malformed CSV falls back to raw text
+        return body.decode("utf-8", errors="replace").strip()
+    rows = [r for r in rows if r]
+    if not rows:
         return ""
+    esc = lambda c: (c or "").replace("|", "\\|")
+    header = "| " + " | ".join(esc(c) for c in rows[0]) + " |"
+    sep = "| " + " | ".join("---" for _ in rows[0]) + " |"
+    lines = [header, sep]
+    for r in rows[1:]:
+        lines.append("| " + " | ".join(esc(c) for c in r) + " |")
+    return "\n".join(lines)
 async def fetch_http(url, max_chars, cookies=None, headers=None, timeout=15):
@@ -143,8 +298,6 @@ async def fetch_http(url, max_chars, cookies=None, headers=None, timeout=15):
                     current = str(httpx.URL(current).join(loc))
                     continue
                 ctype = r.headers.get("content-type", "")
-                if "html" not in ctype and "text" not in ctype:
-                    raise RuntimeError(f"not HTML ({ctype or 'unknown'})")
                 # Read with a hard cap while streaming, so a giant/binary
                 # body cannot exhaust memory. httpx's timeout bounds a
                 # single socket operation only, so a server that slow-drips
@@ -160,15 +313,31 @@ async def fetch_http(url, max_chars, cookies=None, headers=None, timeout=15):
                     if total > MAX_RESPONSE_BYTES:
                         raise ResponseTooLarge(f"response too large (> {MAX_RESPONSE_BYTES} bytes)")
                     chunks.append(chunk)
-                html = b"".join(chunks).decode("utf-8", errors="replace")
+                raw_body = b"".join(chunks)
+                low_ctype = ctype.lower()
+                if "html" in low_ctype or (
+                    "text" in low_ctype and "csv" not in low_ctype and "xml" not in low_ctype
+                ):
+                    html = raw_body.decode("utf-8", errors="replace")
+                else:
+                    title, md, meta = _convert_non_html(ctype, raw_body, current)
+                    return {
+                        "title": title,
+                        "markdown": md[:max_chars],
+                        "metadata": meta,
+                        "non_html": True,
+                        "status_code": r.status_code,
+                        "html": "",
+                    }
                 title = ""
                 m = re.search(r"<title[^>]*>(.*?)</title>", html, re.DOTALL | re.IGNORECASE)
                 if m:
                     title = re.sub(r"\s+", " ", m.group(1)).strip()
-                md = await asyncio.to_thread(_extract_markdown, html)
+                md, meta = await asyncio.to_thread(_extract_with_metadata, html)
                 return {
                     "title": title,
                     "markdown": md[:max_chars],
+                    "metadata": meta,
                     "status_code": r.status_code,
                     "html": html[:8000],
                 }

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/ladder.py RENAMED Viewed

@@ -144,10 +144,16 @@ def _reorder_steps_by_domain(steps, url):
     return [preferred] + [s for s in steps if s != preferred]
+def _empty_meta():
+    """Fresh empty metadata dict (never shared mutable module state)."""
+    return {"author": None, "published_at": None, "site_name": None, "language": None}
 def _normalize_hit(hit):
     return {
         "title": hit.get("title", ""),
         "markdown": hit.get("markdown", ""),
+        "metadata": hit.get("metadata") or _empty_meta(),
         "status": "success",
         "method": "cache",
         "cached": True,
@@ -370,6 +376,7 @@ async def scrape_many(
             results[u] = {
                 "title": "",
                 "markdown": "",
+                "metadata": _empty_meta(),
                 "status": "error",
                 "method": "",
                 "cached": False,
@@ -406,7 +413,11 @@ async def scrape_many(
             reasons[url].append(("error", method, detail))
             return None
         state, authenticated = _auth_state(res, profile)
-        if state == "success" and len((res.get("markdown") or "").strip()) >= 100:
+        # Thin-check guards HTML extraction (an empty/login page misread as
+        # content). Non-HTML conversions (JSON/CSV/feed/PDF) are already
+        # validated by their parser, so a short-but-valid payload counts.
+        thin_ok = len((res.get("markdown") or "").strip()) >= 100 or res.get("non_html")
+        if state == "success" and thin_ok:
             auth = {"profile": profile, "authenticated": authenticated, "state": state}
             # Record which strategy won for this domain so future 'auto'
             # batches can try it first (per-domain strategy memory).
@@ -416,6 +427,7 @@ async def scrape_many(
             out = {
                 "title": res.get("title", ""),
                 "markdown": res.get("markdown", ""),
+                "metadata": res.get("metadata") or _empty_meta(),
                 "status": "success",
                 "method": method,
                 "cached": False,
@@ -460,6 +472,7 @@ async def scrape_many(
                     return url, {
                         "title": "",
                         "markdown": "",
+                        "metadata": _empty_meta(),
                         "status": "error",
                         "method": "http",
                         "cached": False,
@@ -602,6 +615,7 @@ async def scrape_many(
         results[url] = {
             "title": "",
             "markdown": "",
+            "metadata": _empty_meta(),
             "status": state,
             "method": method,
             "cached": False,

{webget_cli-0.11.0 → webget_cli-0.12.0/webget_cli.egg-info}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.11.0
+Version: 0.12.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0
@@ -23,6 +23,7 @@ Requires-Dist: ddgs
 Requires-Dist: httpx
 Requires-Dist: trafilatura
 Requires-Dist: markdownify
+Requires-Dist: pypdf
 Provides-Extra: browser
 Requires-Dist: crawl4ai>=0.9; extra == "browser"
 Provides-Extra: mcp
@@ -188,13 +189,28 @@ path can reuse them. Secrets are never printed.
       "authenticated": true,
       "state": "success"
     },
+    "metadata": {
+      "author": "Jane Doe",
+      "published_at": "2026-09-01",
+      "site_name": "Campus Portal",
+      "language": "id"
+    },
     "error": null
   }
 }
 ```
+`metadata` (author, published date, site name, language) comes from
+trafilatura extraction on the HTTP path; values are `null` when unknown
+or when the winning strategy was not HTTP.
 Status values: `success | login_required | challenge | blocked | error`.
+The HTTP path also handles non-HTML: JSON becomes a pretty code block,
+CSV becomes a GFM table, RSS/Atom feeds become a link list, PDFs are
+extracted per page (via `pypdf`), plain text passes through. Valid
+non-HTML payloads count as success even when short.
 ## Status detection rules
 | Signal | State |

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget_cli.egg-info/SOURCES.txt RENAMED Viewed

@@ -24,6 +24,8 @@ tests/test_mcp_map.py
 tests/test_mcp_profile.py
 tests/test_mcp_server.py
 tests/test_mcp_smoke.py
+tests/test_metadata.py
+tests/test_nonhtml.py
 tests/test_security_review.py
 tests/test_size_review.py
 tests/test_ssrf_dual_dns.py

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget_cli.egg-info/requires.txt RENAMED Viewed

@@ -2,6 +2,7 @@ ddgs
 httpx
 trafilatura
 markdownify
+pypdf
 [browser]
 crawl4ai>=0.9

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget_mcp.py RENAMED Viewed

@@ -230,6 +230,7 @@ async def search_fetch(
                 "snippet": r.get("snippet", ""),
                 "scrape_title": got.get("title", ""),
                 "markdown": got.get("markdown", ""),
+                "metadata": got.get("metadata"),
                 "status": got.get("status", ""),
                 "method": got.get("method", ""),
                 "cached": got.get("cached", False),

{webget_cli-0.11.0 → webget_cli-0.12.0}/LICENSE RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/setup.cfg RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_adversarial_auth.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_adversarial_cache.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_adversarial_concurrency.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_adversarial_http.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_adversarial_mcp.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_adversarial_ssrf.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_auth_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_browser_ssrf.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_cache_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_concurrency_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_discovery_map.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_extraction_markdown.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_firecrawl_policy.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_integration_ladder.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_ladder_retry.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_login_flow.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_mcp_map.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_mcp_profile.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_mcp_server.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_mcp_smoke.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_security_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_size_review.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_ssrf_dual_dns.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/tests/test_webget.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/cache.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/discovery.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/firecrawl.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/profile.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/search.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget/ssrf.py RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget_cli.egg-info/dependency_links.txt RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget_cli.egg-info/entry_points.txt RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget_cli.egg-info/top_level.txt RENAMED Viewed

File without changes

{webget_cli-0.11.0 → webget_cli-0.12.0}/webget_cli.py RENAMED Viewed

File without changes

webget-cli 0.11.0__tar.gz → 0.12.0__tar.gz

webget-cli 0.11.0tar.gz → 0.12.0tar.gz