webget-cli 0.12.0__tar.gz → 0.12.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {webget_cli-0.12.0 → webget_cli-0.12.1}/PKG-INFO +1 -1
  2. {webget_cli-0.12.0 → webget_cli-0.12.1}/pyproject.toml +1 -1
  3. webget_cli-0.12.1/tests/test_base64_strip.py +55 -0
  4. webget_cli-0.12.1/tests/test_mcp_metadata.py +35 -0
  5. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_metadata.py +2 -28
  6. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/http.py +25 -0
  7. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.egg-info/PKG-INFO +1 -1
  8. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.egg-info/SOURCES.txt +2 -0
  9. {webget_cli-0.12.0 → webget_cli-0.12.1}/LICENSE +0 -0
  10. {webget_cli-0.12.0 → webget_cli-0.12.1}/README.md +0 -0
  11. {webget_cli-0.12.0 → webget_cli-0.12.1}/setup.cfg +0 -0
  12. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_adversarial_auth.py +0 -0
  13. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_adversarial_cache.py +0 -0
  14. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_adversarial_concurrency.py +0 -0
  15. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_adversarial_http.py +0 -0
  16. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_adversarial_mcp.py +0 -0
  17. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_adversarial_ssrf.py +0 -0
  18. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_auth_review.py +0 -0
  19. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_browser_ssrf.py +0 -0
  20. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_cache_review.py +0 -0
  21. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_concurrency_review.py +0 -0
  22. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_discovery_map.py +0 -0
  23. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_extraction_markdown.py +0 -0
  24. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_firecrawl_policy.py +0 -0
  25. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_integration_ladder.py +0 -0
  26. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_ladder_retry.py +0 -0
  27. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_login_flow.py +0 -0
  28. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_mcp_leak_review.py +0 -0
  29. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_mcp_map.py +0 -0
  30. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_mcp_profile.py +0 -0
  31. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_mcp_server.py +0 -0
  32. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_mcp_smoke.py +0 -0
  33. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_nonhtml.py +0 -0
  34. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_security_review.py +0 -0
  35. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_size_review.py +0 -0
  36. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_ssrf_dual_dns.py +0 -0
  37. {webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_webget.py +0 -0
  38. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/__init__.py +0 -0
  39. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/cache.py +0 -0
  40. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/cli.py +0 -0
  41. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/discovery.py +0 -0
  42. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/firecrawl.py +0 -0
  43. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/ladder.py +0 -0
  44. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/profile.py +0 -0
  45. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/search.py +0 -0
  46. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget/ssrf.py +0 -0
  47. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.egg-info/dependency_links.txt +0 -0
  48. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.egg-info/entry_points.txt +0 -0
  49. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.egg-info/requires.txt +0 -0
  50. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.egg-info/top_level.txt +0 -0
  51. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.py +0 -0
  52. {webget_cli-0.12.0 → webget_cli-0.12.1}/webget_mcp.py +0 -0
{webget_cli-0.12.0 → webget_cli-0.12.1}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: webget-cli
3
- Version: 0.12.0
3
+ Version: 0.12.1
4
4
  Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
5
5
  Author-email: David Tarigan <tarigansdavid@gmail.com>
6
6
  License: Apache-2.0
{webget_cli-0.12.0 → webget_cli-0.12.1}/pyproject.toml RENAMED
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "webget-cli"
7
- version = "0.12.0"
7
+ version = "0.12.1"
8
8
  description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
webget_cli-0.12.1/tests/test_base64_strip.py ADDED
@@ -0,0 +1,55 @@
1
+ """Ide #3: oversized base64 image payloads are stripped from extracted markdown.
2
+
3
+ Bug scope (probed 2026-09-16, see .hermes/plans/2026-09-16_webget-ide03-base64-strip.md):
4
+ the markdownify fallback keeps inline `data:image/...;base64,...` URLs verbatim,
5
+ so a single hero image can carry hundreds of KB into the output. The trafilatura
6
+ path drops images entirely, so only the fallback needs the strip.
7
+ """
8
+
9
+ import webget_cli as webget
10
+
11
+ BIG_PAYLOAD = "iVBORw0KGgo" + "A" * 800 # > _BASE64_PAYLOAD_MIN
12
+ SMALL_PAYLOAD = "B" * 120 # <= threshold, must survive
13
+
14
+
15
+ def _page(payload):
16
+ return (
17
+ "<html><head><title>T</title></head><body><article><h1>T</h1>"
18
+ f'<p><img alt="hero shot" src="data:image/png;base64,{payload}"></p>'
19
+ "<p>" + ("Body text filler words here to satisfy the length floor. " * 6) + "</p>"
20
+ "</article></body></html>"
21
+ )
22
+
23
+
24
+ def _fallback_only(monkeypatch):
25
+ """Force the markdownify path (trafilatura returns None)."""
26
+ import trafilatura
27
+
28
+ monkeypatch.setattr(trafilatura, "extract", lambda *a, **k: None)
29
+
30
+
31
+ class TestBase64Strip:
32
+ def test_big_payload_stripped_alt_and_mime_survive(self, monkeypatch):
33
+ _fallback_only(monkeypatch)
34
+ text, _ = webget._extract_with_metadata(_page(BIG_PAYLOAD))
35
+ assert BIG_PAYLOAD not in text
36
+ assert "data:image/png;base64,stripped" in text
37
+ assert "hero shot" in text
38
+
39
+ def test_small_payload_preserved(self, monkeypatch):
40
+ _fallback_only(monkeypatch)
41
+ text, _ = webget._extract_with_metadata(_page(SMALL_PAYLOAD))
42
+ assert SMALL_PAYLOAD in text
43
+
44
+ def test_threshold_boundary(self, monkeypatch):
45
+ _fallback_only(monkeypatch)
46
+ at_limit = "C" * 200
47
+ just_over = "D" * 201
48
+ assert at_limit in webget._extract_with_metadata(_page(at_limit))[0]
49
+ assert just_over not in webget._extract_with_metadata(_page(just_over))[0]
50
+
51
+ def test_extract_markdown_contract_unchanged(self, monkeypatch):
52
+ _fallback_only(monkeypatch)
53
+ out = webget._extract_markdown(_page(BIG_PAYLOAD))
54
+ assert isinstance(out, str)
55
+ assert BIG_PAYLOAD not in out
webget_cli-0.12.1/tests/test_mcp_metadata.py ADDED
@@ -0,0 +1,35 @@
1
+ """Ide #1: metadata passthrough in the MCP tool layer.
2
+
3
+ Moved out of tests/test_metadata.py: these import webget_mcp, which needs
4
+ fastmcp/mcp, so they run in the mcp-test CI job (unit job has no fastmcp).
5
+ """
6
+
7
+
8
+ class TestMcpMetadataExposure:
9
+ def test_mcp_fetch_passthrough_has_metadata(self, fresh_cache):
10
+ import asyncio
11
+
12
+ import webget_mcp
13
+
14
+ res = asyncio.run(
15
+ webget_mcp.fetch(fresh_cache.url("/long"), strategy="http", no_cache=True)
16
+ )
17
+ assert set(res["metadata"]) == {"author", "published_at", "site_name", "language"}
18
+
19
+ def test_mcp_search_fetch_rebuild_has_metadata(self, fresh_cache, monkeypatch):
20
+ import asyncio
21
+
22
+ import webget_mcp
23
+
24
+ monkeypatch.setattr(
25
+ webget_mcp.wg,
26
+ "search",
27
+ lambda *a, **k: [{"title": "T", "url": fresh_cache.url("/long"), "snippet": "s"}],
28
+ )
29
+ out = asyncio.run(webget_mcp.search_fetch("q", n=1, no_cache=True))
30
+ assert set(out[0]["metadata"]) == {
31
+ "author",
32
+ "published_at",
33
+ "site_name",
34
+ "language",
35
+ }
{webget_cli-0.12.0 → webget_cli-0.12.1}/tests/test_metadata.py RENAMED
@@ -76,35 +76,9 @@ class TestFetchHttpMetadata:
76
76
  }
77
77
 
78
78
 
79
+ # NOTE: MCP-layer metadata passthrough tests live in tests/test_mcp_metadata.py
80
+ # (they import webget_mcp, which needs fastmcp; that file runs in the mcp-test CI job).
79
81
  class TestOutputExposure:
80
- def test_mcp_fetch_passthrough_has_metadata(self, fresh_cache):
81
- import asyncio
82
-
83
- import webget_mcp
84
-
85
- res = asyncio.run(
86
- webget_mcp.fetch(fresh_cache.url("/long"), strategy="http", no_cache=True)
87
- )
88
- assert set(res["metadata"]) == {"author", "published_at", "site_name", "language"}
89
-
90
- def test_mcp_search_fetch_rebuild_has_metadata(self, fresh_cache, monkeypatch):
91
- import asyncio
92
-
93
- import webget_mcp
94
-
95
- monkeypatch.setattr(
96
- webget_mcp.wg,
97
- "search",
98
- lambda *a, **k: [{"title": "T", "url": fresh_cache.url("/long"), "snippet": "s"}],
99
- )
100
- out = asyncio.run(webget_mcp.search_fetch("q", n=1, no_cache=True))
101
- assert set(out[0]["metadata"]) == {
102
- "author",
103
- "published_at",
104
- "site_name",
105
- "language",
106
- }
107
-
108
82
  def test_cli_su_json_rebuild_has_metadata(self, fresh_cache, monkeypatch, capsys):
109
83
  import webget_cli as wgcli
110
84
  from webget import cli as cli_mod
{webget_cli-0.12.0 → webget_cli-0.12.1}/webget/http.py RENAMED
@@ -35,6 +35,30 @@ class ResponseTooLarge(Exception):
35
35
 
36
36
  _EMPTY_META = {"author": None, "published_at": None, "site_name": None, "language": None}
37
37
 
38
+ # Inline data:image payloads longer than this (base64 chars) are replaced
39
+ # in extracted markdown; shorter ones are left alone since they cost little
40
+ # and may carry meaningful thumbnails.
41
+ _BASE64_PAYLOAD_MIN = 200
42
+ _DATA_URL_IMAGE_RE = re.compile(r"(data:image/[a-z0-9.+-]+;base64,)([A-Za-z0-9+/=]+)")
43
+
44
+
45
+ def _strip_base64_images(text):
46
+ """Replace oversized inline base64 image payloads with 'stripped'.
47
+
48
+ Pages routinely inline images as data URLs; a single hero image can
49
+ carry hundreds of KB of base64, which the markdownify fallback would
50
+ otherwise pass through verbatim as token-burning noise. Keeps the
51
+ mime prefix and the markdown around the URL (alt text lives outside
52
+ the URL), drops only the payload.
53
+ """
54
+
55
+ def _repl(m):
56
+ if len(m.group(2)) <= _BASE64_PAYLOAD_MIN:
57
+ return m.group(0)
58
+ return m.group(1) + "stripped"
59
+
60
+ return _DATA_URL_IMAGE_RE.sub(_repl, text)
61
+
38
62
 
39
63
  def _extract_with_metadata(html):
40
64
  """Extract (text, metadata) from HTML.
@@ -86,6 +110,7 @@ def _extract_with_metadata(html):
86
110
  # output style (verified differential 2026-08-08) so the fallback
87
111
  # stays close to 0.7.2 (semantic parity).
88
112
  converted = md(html, bullets="*", heading_style="ATX").strip()
113
+ converted = _strip_base64_images(converted)
89
114
  if len(converted) > 50:
90
115
  return converted, dict(_EMPTY_META)
91
116
  return "", dict(_EMPTY_META)
{webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: webget-cli
3
- Version: 0.12.0
3
+ Version: 0.12.1
4
4
  Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
5
5
  Author-email: David Tarigan <tarigansdavid@gmail.com>
6
6
  License: Apache-2.0
{webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.egg-info/SOURCES.txt RENAMED
@@ -10,6 +10,7 @@ tests/test_adversarial_http.py
10
10
  tests/test_adversarial_mcp.py
11
11
  tests/test_adversarial_ssrf.py
12
12
  tests/test_auth_review.py
13
+ tests/test_base64_strip.py
13
14
  tests/test_browser_ssrf.py
14
15
  tests/test_cache_review.py
15
16
  tests/test_concurrency_review.py
@@ -21,6 +22,7 @@ tests/test_ladder_retry.py
21
22
  tests/test_login_flow.py
22
23
  tests/test_mcp_leak_review.py
23
24
  tests/test_mcp_map.py
25
+ tests/test_mcp_metadata.py
24
26
  tests/test_mcp_profile.py
25
27
  tests/test_mcp_server.py
26
28
  tests/test_mcp_smoke.py
{webget_cli-0.12.0 → webget_cli-0.12.1}/LICENSE RENAMED
File without changes
{webget_cli-0.12.0 → webget_cli-0.12.1}/README.md RENAMED
File without changes
{webget_cli-0.12.0 → webget_cli-0.12.1}/setup.cfg RENAMED
File without changes
{webget_cli-0.12.0 → webget_cli-0.12.1}/webget/cache.py RENAMED
File without changes
{webget_cli-0.12.0 → webget_cli-0.12.1}/webget/cli.py RENAMED
File without changes
{webget_cli-0.12.0 → webget_cli-0.12.1}/webget/ssrf.py RENAMED
File without changes
{webget_cli-0.12.0 → webget_cli-0.12.1}/webget_cli.py RENAMED
File without changes
{webget_cli-0.12.0 → webget_cli-0.12.1}/webget_mcp.py RENAMED
File without changes