webget-cli 0.9.0__tar.gz → 0.10.0__tar.gz

{webget_cli-0.9.0 → webget_cli-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webget-cli
-Version: 0.9.0
+Version: 0.10.0
 Summary: Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys.
 Author-email: David Tarigan <tarigansdavid@gmail.com>
 License: Apache-2.0

{webget_cli-0.9.0 → webget_cli-0.10.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webget-cli"
-version = "0.9.0"
+version = "0.10.0"
 description = "Local search + scrape CLI with an HTTP fast path, optional browser fallback, and authenticated session profiles. Zero API keys."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -39,11 +39,12 @@ mcp = ["fastmcp>=2"]
 dev = ["pytest>=8", "ruff>=0.6"]
 
 [project.scripts]
-webget = "webget_cli:main"
+webget = "webget.cli:main"
 webget-mcp = "webget_mcp:main"
 
 [tool.setuptools]
 py-modules = ["webget_cli", "webget_mcp"]
+packages = ["webget"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

{webget_cli-0.9.0 → webget_cli-0.10.0}/tests/test_adversarial_mcp.py

@@ -58,7 +58,7 @@ class TestMalformedArguments:
                 return res
 
         res = _run(run())
-        assert res.is_error
+        assert res.isError
 
 
 class TestInvalidURLs:
@@ -72,7 +72,7 @@ class TestInvalidURLs:
                 return res
 
         res = _run(run())
-        assert "error" in res.content[0].text or res.is_error
+        assert "error" in res.content[0].text or res.isError
 
 
 class TestRepeatedCalls:
@@ -86,7 +86,7 @@ class TestRepeatedCalls:
                         "fetch",
                         {"url": "https://example.com", "strategy": "http", "no_cache": True},
                     )
-                    if res.is_error:
+                    if res.isError:
                         return "ERROR"
                 return "OK"
 
@@ -106,7 +106,7 @@ class TestRepeatedCalls:
                         for _ in range(5)
                     ]
                 )
-                return [r.is_error for r in results]
+                return [r.isError for r in results]
 
         assert _run(run()) == [False] * 5
 
@@ -133,7 +133,7 @@ class TestInputCaps:
                 return res
 
         res = _run(run())
-        assert "must be between" in res.content[0].text or res.is_error
+        assert "must be between" in res.content[0].text or res.isError
 
 
 class TestToolFailureIsolation:

{webget_cli-0.9.0 → webget_cli-0.10.0}/tests/test_login_flow.py

@@ -133,7 +133,7 @@ def test_mcp_login_tool_persists_and_fetch_uses_it(server, tmp_path):
 
     res, authed, anon = asyncio.run(asyncio.wait_for(run(), timeout=90))
     res_text = res.content[0].text if res.content else ""
-    assert not res.is_error, res_text
+    assert not res.isError, res_text
     assert '"status":"success"' in res_text, res_text
     assert '"profile":"mcplogin"' in res_text
 

{webget_cli-0.9.0 → webget_cli-0.10.0}/tests/test_mcp_leak_review.py

@@ -64,7 +64,7 @@ class TestNoSecretLeakage:
                 return res
 
         res = _run(run())
-        if res.is_error:
+        if res.isError:
             return  # error path: no payload to leak, still fine
         text = res.content[0].text
         hits = _scan(text)
@@ -121,7 +121,7 @@ class TestServerRecovery:
                 bad = await session.call_tool(
                     "fetch", {"url": "https://example.com", "strategy": "bogus"}
                 )
-                assert not bad.is_error or "error" in bad.content[0].text
+                assert not bad.isError or "error" in bad.content[0].text
                 good = await session.call_tool(
                     "fetch", {"url": "https://example.com", "strategy": "http", "no_cache": True}
                 )

{webget_cli-0.9.0 → webget_cli-0.10.0}/tests/test_mcp_profile.py

@@ -79,7 +79,7 @@ def test_mcp_fetch_with_profile_uses_session(tmp_path):
         ok, anon = asyncio.run(asyncio.wait_for(run(), timeout=60))
         ok_text = ok.content[0].text if ok.content else ""
         anon_text = anon.content[0].text if anon.content else ""
-        assert not ok.is_error, ok_text
+        assert not ok.isError, ok_text
         assert '"status":"success"' in ok_text, ok_text
         assert "you are authenticated" in ok_text, ok_text
         assert '"authenticated":true' in ok_text, ok_text  # session was used
@@ -105,7 +105,7 @@ def test_mcp_fetch_unknown_profile_is_hard_error(tmp_path):
             return res
 
     res = asyncio.run(asyncio.wait_for(run(), timeout=60))
-    assert not res.is_error
+    assert not res.isError
     assert "profile 'ghost' not found" in res.content[0].text
 
 
@@ -150,7 +150,7 @@ def test_mcp_fetch_enriched_output_present(tmp_path):
         anon_text = anon.content[0].text if anon.content else ""
 
         # Authenticated: standard assertions
-        assert not ok.is_error, ok_text
+        assert not ok.isError, ok_text
         assert '"status":"success"' in ok_text
         # auth
         assert '"auth"' in ok_text

{webget_cli-0.9.0 → webget_cli-0.10.0}/tests/test_mcp_server.py

@@ -53,7 +53,7 @@ def test_invalid_strategy_returns_error_not_crash():
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "error" in res.content[0].text
 
 
@@ -72,7 +72,7 @@ def test_firecrawl_without_key_returns_error_not_crash():
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "error" in res.content[0].text
 
 
@@ -95,7 +95,7 @@ def test_server_stays_alive_after_bad_calls():
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "success" in res.content[0].text
 
 
@@ -118,7 +118,7 @@ def test_fetch_invalid_profile_returns_error_not_crash():
             return res, ok
 
     res, ok = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "invalid profile name" in res.content[0].text
     assert "success" in ok.content[0].text  # server alive after the bad call
 
@@ -139,7 +139,7 @@ def test_fetch_nonexistent_profile_returns_error():
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "profile 'ghost' not found" in res.content[0].text
 
 
@@ -161,9 +161,9 @@ def test_search_fetch_invalid_profile_returns_error_not_crash():
             return res, ok
 
     res, ok = _run(run())
-    assert not res.is_error
+    assert not res.isError
     assert "invalid profile name" in res.content[0].text
-    assert not ok.is_error  # server alive after the bad call
+    assert not ok.isError  # server alive after the bad call
 
 
 def test_list_profiles_tool_metadata_only(tmp_path):
@@ -189,7 +189,7 @@ def test_list_profiles_tool_metadata_only(tmp_path):
             return res
 
     res = _run(run())
-    assert not res.is_error
+    assert not res.isError
     text = res.content[0].text
     assert "sion" in text
     assert "SUPERSECRET" not in text  # cookie values never exposed

webget_cli-0.10.0/webget/__init__.py

@@ -0,0 +1,144 @@
+"""webget - local search + scrape, zero API keys, unlimited usage.
+
+A package refactor of the original single-file webget_cli.py. Public API
+is re-exported here so callers can do `from webget import fetch_http,
+scrape_many, ...` instead of importing internal modules.
+
+Sub-modules:
+  - cache     - disk cache + cookie/header parsing
+  - ssrf      - private-IP guard for HTTP + browser paths
+  - profile   - persistent browser-profile (auth session) management
+  - http      - fast-path HTTP fetch + markdown extraction
+  - firecrawl - optional cloud escape-hatch fetch
+  - search    - DuckDuckGo text search + atomic JSON helpers
+  - ladder    - HTTP -> Crawl4AI -> Firecrawl orchestration
+  - cli       - argparse entry point
+
+The legacy single-file import `import webget_cli as webget` keeps
+working via the compatibility shim at ./webget_cli.py.
+"""
+from __future__ import annotations
+
+from .cache import (
+    _CACHE_SWEEP_TTL,
+    CACHE_DIR,
+    _cache_path,
+    cache_get,
+    cache_put,
+    parse_cookie_file,
+    parse_headers,
+)
+from .cli import main, parse_opts
+from .firecrawl import fetch_firecrawl, firecrawl_key
+from .http import MAX_RESPONSE_BYTES, ResponseTooLarge, _extract_markdown, fetch_http
+from .ladder import (
+    _DEFAULT_CONCURRENCY,
+    _STRATEGY_MEMORY_TTL,
+    _auth_message,
+    _crawl4ai_once,
+    _ladder,
+    _learn_strategy,
+    _load_strategy_memory,
+    _normalize_hit,
+    _reorder_steps_by_domain,
+    _save_strategy_memory,
+    _strategy_memory_path,
+    _terminal_state,
+    scrape_many,
+)
+from .profile import (
+    _PROFILE_NAME_RE,
+    PROFILE_DIR,
+    _auth_state,
+    _cookie_belongs_to,
+    _domain_match,
+    _effective_cookies,
+    _fmt_age,
+    _login_flow,
+    _logout_domain_regex,
+    _logout_flow,
+    _profile_meta,
+    _profile_root,
+    _prune_storage_cookies,
+    _valid_site_url,
+    _wait_for_session_cookies,
+    _warn,
+    list_profiles,
+    load_profile_cookies,
+    profile_dir,
+    profile_exists,
+    profile_state_path,
+)
+from .search import _read_json, _write_json, search
+from .ssrf import (
+    SSRFError,
+    _guard_browser_routes,
+    _hostname_private,
+    _ip_is_private,
+    _is_private_target,
+    _private_ip_for,
+    _request_body_bytes,
+)
+
+# Sorted to satisfy ruff RUF022; module grouping lives in the imports above.
+__all__ = [
+    "CACHE_DIR",
+    "MAX_RESPONSE_BYTES",
+    "PROFILE_DIR",
+    "_CACHE_SWEEP_TTL",
+    "_DEFAULT_CONCURRENCY",
+    "_PROFILE_NAME_RE",
+    "_STRATEGY_MEMORY_TTL",
+    "ResponseTooLarge",
+    "SSRFError",
+    "_auth_message",
+    "_auth_state",
+    "_cache_path",
+    "_cookie_belongs_to",
+    "_crawl4ai_once",
+    "_domain_match",
+    "_effective_cookies",
+    "_extract_markdown",
+    "_fmt_age",
+    "_guard_browser_routes",
+    "_hostname_private",
+    "_ip_is_private",
+    "_is_private_target",
+    "_ladder",
+    "_learn_strategy",
+    "_load_strategy_memory",
+    "_login_flow",
+    "_logout_domain_regex",
+    "_logout_flow",
+    "_normalize_hit",
+    "_private_ip_for",
+    "_profile_meta",
+    "_profile_root",
+    "_prune_storage_cookies",
+    "_read_json",
+    "_reorder_steps_by_domain",
+    "_request_body_bytes",
+    "_save_strategy_memory",
+    "_strategy_memory_path",
+    "_terminal_state",
+    "_valid_site_url",
+    "_wait_for_session_cookies",
+    "_warn",
+    "_write_json",
+    "cache_get",
+    "cache_put",
+    "fetch_firecrawl",
+    "fetch_http",
+    "firecrawl_key",
+    "list_profiles",
+    "load_profile_cookies",
+    "main",
+    "parse_cookie_file",
+    "parse_headers",
+    "parse_opts",
+    "profile_dir",
+    "profile_exists",
+    "profile_state_path",
+    "scrape_many",
+    "search",
+]

webget_cli-0.10.0/webget/cache.py

@@ -0,0 +1,187 @@
+"""Cache + cookie/header parsing for webget.
+
+Disk cache lives at CACHE_DIR and is keyed by (profile, url, max_chars,
+sorted cookies, sorted headers) so semantically-identical states share one
+entry. Writes are atomic (per-writer unique tmp + os.replace) and the cache
+self-evicts via two-phase sweep when over the cap.
+
+CACHE_DIR is resolved lazily through a getter that consults the
+top-level webget package first, falling back to the package default.
+This keeps `monkeypatch.setattr(webget_cli, "CACHE_DIR", ...)` working
+in the test suite: the canonical location for the path is the shim's
+module attribute, and package sub-modules follow whatever the shim
+currently exposes.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+import time
+
+# Package default; tests that patch webget_cli.CACHE_DIR take precedence
+# because _cache_dir() looks up the parent package's attribute at call time.
+_CACHE_DIR_DEFAULT = os.path.expanduser("~/.cache/webget")
+
+
+def _cache_dir():
+    """Resolve the active cache directory, honoring runtime overrides.
+
+    Tests patch `webget_cli.CACHE_DIR` (the shim module attribute) to
+    point at a tmp dir. Sub-modules in the package consult THIS function
+    so the same override applies everywhere without circular imports.
+    """
+    try:
+        import webget_cli as _shim
+
+        return _shim.CACHE_DIR
+    except (ImportError, AttributeError):
+        return _CACHE_DIR_DEFAULT
+
+
+# Backwards-compatible module attr: defaults to the package default, but
+# assigning to `webget.cache.CACHE_DIR` only affects THIS module. Tests
+# use `webget_cli.CACHE_DIR` (the shim) as the override hook, which is
+# picked up via _cache_dir() above.
+CACHE_DIR = _CACHE_DIR_DEFAULT
+
+# Entries older than this can never be served by any reasonable ttl and are
+# swept on write. 30 days matches the default CLI ttl.
+_CACHE_SWEEP_TTL = 3600 * 24 * 30
+
+
+def parse_cookie_file(path):
+    """Parse Netscape cookie file -> Playwright-compatible cookie dicts."""
+    cookies = []
+    with open(os.path.expanduser(path)) as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            parts = line.split("\t")
+            if len(parts) < 7:
+                continue
+            domain, _domain_flag, path_, secure, expires, name, value = parts[:7]
+            cookie = {
+                "name": name,
+                "value": value,
+                "domain": domain,
+                "path": path_,
+                "secure": secure == "TRUE",
+                "httpOnly": False,
+            }
+            if expires not in ("0", "Session", ""):
+                try:
+                    cookie["expires"] = int(expires)
+                except ValueError:
+                    pass
+            cookies.append(cookie)
+    return cookies
+
+
+def parse_headers(raw_headers):
+    """Parse list of 'Key: Value' strings into a dict."""
+    headers = {}
+    for h in raw_headers:
+        if ":" in h:
+            k, v = h.split(":", 1)
+            headers[k.strip()] = v.strip()
+    return headers
+
+
+def _cache_path(url, cookies, headers, max_chars, profile=None):
+    # Cookies and headers are sorted so semantically identical states (same
+    # cookie set in different order) share one cache entry.
+    ck = sorted(cookies or [], key=lambda c: (c.get("domain", ""), c.get("name", "")))
+    hd = {k: v for k, v in sorted((headers or {}).items())}
+    key = hashlib.sha1(
+        f"{profile or 'public'}|{url}|{max_chars}|"
+        f"{json.dumps(ck, sort_keys=True)}|"
+        f"{json.dumps(hd, sort_keys=True)}".encode()
+    ).hexdigest()
+    return os.path.join(_cache_dir(), key + ".json")
+
+
+def cache_get(url, cookies, headers, max_chars, ttl, fresh=False, profile=None):
+    if fresh:
+        return None
+    p = _cache_path(url, cookies, headers, max_chars, profile)
+    if not os.path.exists(p):
+        return None
+    if time.time() - os.path.getmtime(p) > ttl:
+        return None
+    try:
+        with open(p) as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def cache_put(url, cookies, headers, max_chars, data, profile=None):
+    cache_dir = _cache_dir()
+    os.makedirs(cache_dir, exist_ok=True)
+    p = _cache_path(url, cookies, headers, max_chars, profile)
+    tmp = None
+    try:
+        # Atomic write: each writer gets its OWN unique tmp file (mkstemp),
+        # then an atomic rename. A shared "<path>.tmp" would let concurrent
+        # writers interleave bytes in the same file, so the rename would
+        # publish a corrupt document. mkstemp guarantees uniqueness per
+        # writer and lives in the same directory (same filesystem, so
+        # os.replace stays atomic).
+        fd, tmp = tempfile.mkstemp(
+            dir=os.path.dirname(p), prefix=os.path.basename(p) + ".", suffix=".tmp"
+        )
+        with os.fdopen(fd, "w") as f:
+            json.dump({**data, "fetched_at": time.time()}, f)
+        os.replace(tmp, p)
+        tmp = None  # consumed by the rename
+        # Eviction (lazy: only scans when actually over the cap, so the hot
+        # write path stays cheap). Drop expired entries first - they can
+        # never be served - then keep newest 80% if still over. The cap is
+        # tunable via WEBGET_CACHE_MAX because a 500-file ceiling makes
+        # bulk crawls (thousands of URLs) thrash: every write evicts
+        # useful entries.
+        try:
+            cap = int(os.environ.get("WEBGET_CACHE_MAX", "5000"))
+        except ValueError:
+            cap = 5000
+        try:
+            files = [
+                os.path.join(cache_dir, f)
+                for f in os.listdir(cache_dir)
+                if f.endswith(".json") and f != "strategy_memory.json"
+            ]
+            if len(files) > cap:
+                now = time.time()
+                live = []
+                for fp in files:
+                    try:
+                        age = now - os.path.getmtime(fp)
+                    except OSError:
+                        continue
+                    if age > _CACHE_SWEEP_TTL:
+                        try:
+                            os.remove(fp)
+                        except OSError:
+                            pass
+                    else:
+                        live.append(fp)
+                if len(live) > cap:
+                    live.sort(key=os.path.getmtime)
+                    for f in live[: len(live) - int(cap * 0.8)]:
+                        try:
+                            os.remove(f)
+                        except OSError:
+                            pass
+        except OSError:
+            pass
+    except OSError:
+        pass  # cache is best-effort
+    finally:
+        if tmp is not None:
+            try:
+                os.remove(tmp)
+            except OSError:
+                pass