agent-wire-cli 0.2.1__py3-none-any.whl

agent_wire/compression/bm25_slicer.py

@@ -0,0 +1,151 @@
+"""Query-aware BM25 + social-signal block ranking that packs the best blocks into a token budget."""
+
+import math
+import os
+import re
+from collections import Counter
+from functools import cache
+
+import tiktoken
+
+from agent_wire.config import init_wire_home
+from agent_wire.models import WireComment
+
+HEADER_TOKENS = 200
+K1, B = 1.5, 0.75
+W_ACCEPTED, W_OP = 3.0, 1.5
+
+_WORD = re.compile(r"\w+")
+_HEADING_ONLY = re.compile(r"#{1,6}\s+[^\n]*")
+
+
+@cache
+def _encoding() -> tiktoken.Encoding:
+    # tiktoken caches its BPE file in the OS temp dir by default; keep it in WIRE_HOME instead.
+    cache_dir = init_wire_home() / "tiktoken"
+    os.environ.setdefault("TIKTOKEN_CACHE_DIR", str(cache_dir))
+    try:
+        return tiktoken.get_encoding("o200k_base")
+    except FileExistsError:
+        # Several first runs at once (clients starting their servers together) each download the
+        # file and rename it into place; on Windows the losers' rename fails. The winner's copy is
+        # there now, so read it, and drop the losers' temp files.
+        for tmp in cache_dir.glob("*.tmp"):
+            tmp.unlink(missing_ok=True)
+        return tiktoken.get_encoding("o200k_base")
+
+
+def count_tokens(text: str) -> int:
+    return len(_encoding().encode(text, disallowed_special=()))
+
+
+def truncate_tokens(text: str, max_tokens: int) -> str:
+    tokens = _encoding().encode(text, disallowed_special=())
+    return text if len(tokens) <= max_tokens else _encoding().decode(tokens[:max_tokens])
+
+
+def split_blocks(text: str) -> list[str]:
+    """Paragraph blocks; fenced code stays whole and a lone heading joins the block after it."""
+    blocks, buf, fence = [], [], None
+    for line in text.splitlines():
+        stripped = line.lstrip()
+        if fence:
+            buf.append(line)
+            if stripped.startswith(fence):
+                fence = None
+        elif stripped.startswith(("```", "~~~")):
+            fence = stripped[:3]
+            buf.append(line)
+        elif not stripped:
+            if buf:
+                blocks.append("\n".join(buf))
+                buf = []
+        else:
+            buf.append(line)
+    if buf:
+        blocks.append("\n".join(buf))
+
+    merged, heading = [], None
+    for block in blocks:
+        if heading:
+            block, heading = f"{heading}\n\n{block}", None
+        if _HEADING_ONLY.fullmatch(block):
+            heading = block
+        else:
+            merged.append(block)
+    if heading:
+        merged.append(heading)
+    return merged
+
+
+def bm25_scores(blocks: list[str], query: str) -> list[float]:
+    terms = set(_WORD.findall(query.lower()))
+    if not terms or not blocks:
+        return [0.0] * len(blocks)
+    docs = [Counter(_WORD.findall(b.lower())) for b in blocks]
+    n = len(docs)
+    avgdl = sum(sum(d.values()) for d in docs) / n or 1.0
+    idf = {}
+    for t in terms:
+        df = sum(1 for d in docs if t in d)
+        idf[t] = math.log((n - df + 0.5) / (df + 0.5) + 1)
+    scores = []
+    for d in docs:
+        dl = sum(d.values())
+        s = 0.0
+        for t in terms:
+            if tf := d.get(t):
+                s += idf[t] * tf * (K1 + 1) / (tf + K1 * (1 - B + B * dl / avgdl))
+        scores.append(s)
+    return scores
+
+
+def select(costs: list[int], scores: list[float], budget: int) -> list[int]:
+    """Greedy pack by score (ties keep document order); returns kept indices in document order."""
+    kept, used = [], 0
+    for i in sorted(range(len(costs)), key=lambda i: -scores[i]):
+        if used + costs[i] <= budget:
+            kept.append(i)
+            used += costs[i]
+    return sorted(kept)
+
+
+def slice_text(text: str, query: str | None = None, max_tokens: int = 3000) -> str:
+    if count_tokens(text) <= max_tokens:
+        return text
+    blocks = split_blocks(text)
+    if len(blocks) == 1:
+        return truncate_tokens(text, max_tokens)
+
+    # Always keep the lead (title / TL;DR / first ~200 tokens), even if it isn't query-relevant.
+    lead_budget = min(HEADER_TOKENS, max_tokens)
+    costs = [count_tokens(b) + 1 for b in blocks]  # +1 for the joining blank line
+    lead, used = 0, 0
+    while lead < len(blocks) and used + costs[lead] <= lead_budget:
+        used += costs[lead]
+        lead += 1
+    if lead == 0:
+        blocks[0] = truncate_tokens(blocks[0], lead_budget)
+        used, lead = count_tokens(blocks[0]) + 1, 1
+
+    rest = blocks[lead:]
+    kept = select(costs[lead:], bm25_scores(rest, query or ""), max_tokens - used)
+    return "\n\n".join(blocks[:lead] + [rest[i] for i in kept])
+
+
+def _subtree_text(c: WireComment) -> str:
+    return "\n".join([c.body, *(_subtree_text(r) for r in c.replies)])
+
+
+def rank_comments(
+    comments: list[WireComment], query: str | None = None, max_tokens: int = 1500
+) -> list[WireComment]:
+    """Keep the highest-signal top-level comment sub-trees that fit, in original order."""
+    texts = [_subtree_text(c) for c in comments]
+    bm = bm25_scores(texts, query or "")
+    scores = [
+        bm[i] + W_ACCEPTED * c.is_accepted + W_OP * c.is_op + math.log1p(max(0, c.score))
+        for i, c in enumerate(comments)
+    ]
+    kept = select([count_tokens(t) for t in texts], scores, max_tokens)
+    return [comments[i] for i in kept]

agent_wire/compression/dam_formatter.py

@@ -0,0 +1,74 @@
+"""Dense Agent Markdown: compact document + comment-tree rendering and the [WIRE | ...] envelope."""
+
+import re
+from datetime import UTC, datetime
+
+from agent_wire.compression.url_compactor import compact
+from agent_wire.firewall.inbound_shield import spotlight
+from agent_wire.models import WireComment, WireDocument, WireTelemetry
+
+_IMAGE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
+_BLANKS = re.compile(r"\n{3,}")
+_AGE_UNITS = (("y", 31_536_000), ("mo", 2_592_000), ("d", 86_400), ("h", 3_600), ("m", 60))
+
+
+def telemetry_header(t: WireTelemetry) -> str:
+    return (
+        f"[WIRE | {t.channel} | {t.tokens} tokens (saved {t.saved_pct}%) | auth:{t.auth}"
+        f" | stage:{t.stage} | {t.elapsed_ms}ms]"
+    )
+
+
+def envelope(t: WireTelemetry, content: str) -> str:
+    return spotlight(telemetry_header(t), content)
+
+
+def relative_age(ts: datetime, now: datetime | None = None) -> str:
+    now = now or datetime.now(UTC)
+    if ts.tzinfo is None:
+        ts = ts.replace(tzinfo=UTC)
+    seconds = max(0, int((now - ts).total_seconds()))
+    for unit, size in _AGE_UNITS:
+        if seconds >= size:
+            return f"{seconds // size}{unit} ago"
+    return "just now"
+
+
+def format_comments(
+    comments: list[WireComment], now: datetime | None = None, depth: int = 0
+) -> str:
+    lines = []
+    prefix = "  " * depth + ("> " if depth else "")
+    for c in comments:
+        tags = (
+            [f"{c.score} replies" if c.score_kind == "replies" else f"{c.score:+d}"]
+            + ["OP"] * c.is_op
+            + ["ACCEPTED"] * c.is_accepted
+        )
+        who = f"@{c.author_handle}" if c.author_handle else "@[deleted]"
+        age = f" ({relative_age(c.timestamp, now)})" if c.timestamp else ""
+        body = [ln for ln in c.body.strip().splitlines() if ln.strip()] or [""]
+        lines.append(f"{prefix}[{' | '.join(tags)}] {who}{age}: {body[0]}")
+        lines.extend(" " * len(prefix) + ln for ln in body[1:])
+        if c.replies:
+            lines.append(format_comments(c.replies, now, depth + 1))
+    return "\n".join(lines)
+
+
+def render_document(doc: WireDocument, now: datetime | None = None) -> str:
+    parts = [f"# {doc.title}"] if doc.title else []
+    meta = []
+    if doc.author_handle:
+        meta.append(f"@{doc.author_handle}")
+    if doc.timestamp:
+        meta.append(doc.timestamp.date().isoformat())
+    if doc.score is not None:
+        meta.append(f"{doc.score:+d}")
+    meta += [f"{k}: {v}" for k, v in doc.metadata.items() if v is not None]
+    if meta:
+        parts.append(" · ".join(meta))
+    if body := _IMAGE.sub("", doc.body).strip():
+        parts.append(body)
+    if doc.comments:
+        parts.append("## Comments\n" + format_comments(doc.comments, now))
+    return compact(_BLANKS.sub("\n\n", "\n\n".join(parts)))

agent_wire/compression/url_compactor.py

@@ -0,0 +1,45 @@
+"""Strip tracking params from URLs and fold repeated Markdown links into reference links."""
+
+import re
+from collections import Counter
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+_TRACKING = re.compile(r"utm_\w*|fbclid|gclid|si|ref|spm", re.IGNORECASE)
+_URL = re.compile(r"https?://[^\s<>()\[\]\"']+")
+_MD_LINK = re.compile(r"(?<!!)\[([^\]]*)\]\((\S+?)\)")
+
+
+def clean_url(url: str) -> str:
+    parts = urlsplit(url)
+    if not parts.query:
+        return url
+    pairs = parse_qsl(parts.query, keep_blank_values=True)
+    kept = [(k, v) for k, v in pairs if not _TRACKING.fullmatch(k)]
+    if len(kept) == len(pairs):
+        return url  # untouched: avoid re-encoding churn
+    return urlunsplit(parts._replace(query=urlencode(kept)))
+
+
+def _clean_match(m: re.Match) -> str:
+    url = m[0].rstrip(".,;:!?")
+    return clean_url(url) + m[0][len(url) :]
+
+
+def compress_links(markdown: str) -> str:
+    counts = Counter(m[2] for m in _MD_LINK.finditer(markdown))
+    if not any(n > 1 for n in counts.values()):
+        return markdown
+    ids: dict[str, int] = {}
+
+    def repl(m: re.Match) -> str:
+        if counts[m[2]] < 2:
+            return m[0]
+        return f"[{m[1]}][{ids.setdefault(m[2], len(ids) + 1)}]"
+
+    body = _MD_LINK.sub(repl, markdown)
+    refs = "\n".join(f"[{n}]: {url}" for url, n in ids.items())
+    return f"{body}\n\n{refs}"
+
+
+def compact(text: str) -> str:
+    return compress_links(_URL.sub(_clean_match, text))

agent_wire/config.py

@@ -0,0 +1,52 @@
+"""Portable WIRE_HOME resolution and runtime profiles."""
+
+import os
+import sys
+from pathlib import Path
+
+CLIENT_DIRS = (".claude", ".codex", ".hermes", ".agent", ".cursor", ".windsurf", ".roo", ".gemini")
+
+# Everything in WIRE_HOME (vault.enc, cache.db, venvs) stays out of git except config.
+WIRE_GITIGNORE = "*\n!.gitignore\n!config.toml\n"
+
+PROFILES = {
+    "standard": {"read_max_tokens": 3000, "search_max_tokens": 2500, "auto_expand_top": 2},
+    # IDEA says auto_expand_top=1.5; a result count must be whole, so 1.
+    "local": {"read_max_tokens": 800, "search_max_tokens": 800, "auto_expand_top": 1},
+}
+
+
+def _client_dir(path: Path) -> Path | None:
+    for p in (path, *path.parents):
+        if p.name in CLIENT_DIRS:
+            return p
+    return None
+
+
+def resolve_wire_home() -> Path:
+    if env := os.environ.get("WIRE_HOME"):
+        return Path(env)
+    if root := os.environ.get("CLAUDE_PLUGIN_ROOT"):
+        return Path(root) / ".wire"
+    # Run from inside a client folder, or installed into one (venv lives under it).
+    for start in (Path.cwd(), Path(__file__).resolve()):
+        if client := _client_dir(start):
+            return client / ".wire"
+    if sys.platform == "win32":
+        return Path(os.environ.get("APPDATA") or Path.home() / "AppData" / "Roaming") / "agent-wire"
+    return Path.home() / ".config" / "agent-wire"
+
+
+def init_wire_home(home: Path | None = None) -> Path:
+    home = home or resolve_wire_home()
+    home.mkdir(parents=True, exist_ok=True)
+    gitignore = home / ".gitignore"
+    if not gitignore.exists():
+        gitignore.write_text(WIRE_GITIGNORE, encoding="utf-8")
+    return home
+
+
+def get_profile(name: str = "standard") -> dict[str, int]:
+    if name not in PROFILES:
+        raise ValueError(f"unknown profile {name!r}; expected one of {sorted(PROFILES)}")
+    return PROFILES[name]

agent_wire/engine/challenge.py

@@ -0,0 +1,58 @@
+"""Detect Cloudflare / DataDome / Turnstile challenges, login walls and empty JS shells."""
+
+from urllib.parse import urlsplit
+
+from selectolax.lexbor import LexborHTMLParser
+
+from agent_wire.engine.http_pool import Fetched
+
+BLOCK_STATUSES = frozenset({401, 403, 429, 503})
+SMALL_BODY = 35_000
+MIN_VISIBLE_CHARS = 80
+
+# Only ever present on a challenge page, so safe to match anywhere near the top.
+_HARD = (
+    b"<title>just a moment...</title>",
+    b"cf-browser-verification",
+    b"cf_chl_opt",
+    b"captcha-delivery.com",  # DataDome block page (its normal tag script lives on js.datadome.co)
+)
+# Also appear on ordinary pages (articles about them, login forms), so only on small bodies.
+_SOFT = (
+    b"challenges.cloudflare.com/turnstile",
+    b"<title>access denied</title>",
+    b"enable javascript and cookies to continue",
+)
+_LOGIN_PATHS = ("/login", "/signin", "/sign-in", "/accounts/login", "/i/flow/login", "/authwall")
+
+
+def visible_text_len(html: bytes) -> int:
+    tree = LexborHTMLParser(html)
+    for node in tree.css("script, style, noscript, template"):
+        node.decompose()
+    body = tree.body
+    return len(body.text(strip=True)) if body else 0
+
+
+def is_blocked(f: Fetched, *, rendered: bool = False) -> bool:
+    """`rendered=True` for stage-4 output: JS already ran, so a short page is just short."""
+    small = len(f.content) < SMALL_BODY
+    if f.status in BLOCK_STATUSES and small:
+        return True
+    head = f.content[:65_536].lower()
+    if any(s in head for s in _HARD):
+        return True
+    if small and any(s in head for s in _SOFT):
+        return True
+    return bool(
+        not rendered
+        and f.ok
+        and f.content
+        and "html" in f.content_type
+        and visible_text_len(f.content) < MIN_VISIBLE_CHARS
+    )
+
+
+def is_auth_wall(f: Fetched) -> bool:
+    path = urlsplit(f.url).path.lower()
+    return f.status in (401, 403) or any(path.startswith(p) for p in _LOGIN_PATHS)

agent_wire/engine/http_pool.py

@@ -0,0 +1,136 @@
+"""Persistent curl_cffi connection pool with browser TLS / HTTP2 fingerprints."""
+
+from dataclasses import dataclass
+from email.message import Message
+from urllib.parse import urljoin, urlsplit
+
+import orjson
+from curl_cffi.requests import AsyncSession
+
+from agent_wire import __version__
+from agent_wire.firewall.outbound_privacy import check_request
+
+DEFAULT_TIMEOUT = 12  # a stalled stage should fail fast and let the waterfall escalate
+USER_AGENT = f"agent-wire/{__version__} (+https://github.com/uncoalesced/agent-wire)"
+MAX_COOKIE_REDIRECTS = 5
+_TWO_LEVEL_TLDS = frozenset(
+    {"co.uk", "org.uk", "ac.uk", "com.au", "co.in", "co.jp", "com.cn", "com.br", "co.kr", "com.tw"}
+)
+
+
+def registrable_domain(host: str | None) -> str:
+    # shortcut: tiny 2-level TLD list instead of the Public Suffix List; upgrade if mis-grouping shows up.
+    labels = (host or "").lower().rstrip(".").split(".")
+    n = 3 if ".".join(labels[-2:]) in _TWO_LEVEL_TLDS else 2
+    return ".".join(labels[-n:])
+
+
+@dataclass
+class Fetched:
+    url: str
+    status: int
+    headers: dict[str, str]
+    content: bytes
+    requested: str = ""
+    stage: int = 1
+    auth: str = "none"
+
+    @property
+    def ok(self) -> bool:
+        return 200 <= self.status < 300
+
+    @property
+    def content_type(self) -> str:
+        return self.headers.get("content-type", "").split(";")[0].strip().lower()
+
+    @property
+    def text(self) -> str:
+        msg = Message()
+        msg["content-type"] = self.headers.get("content-type", "")
+        charset = msg.get_param("charset") or "utf-8"
+        try:
+            return self.content.decode(str(charset), errors="replace")
+        except LookupError:
+            return self.content.decode("utf-8", errors="replace")
+
+    def json(self):
+        return orjson.loads(self.content)
+
+
+class HttpPool:
+    """One warm AsyncSession per fingerprint for anonymous traffic.
+
+    Cookie-bearing requests get a throwaway session so the user's session cookies (and anything
+    the site sets in reply) never end up in the shared anonymous cookie jars.
+    """
+
+    def __init__(self, timeout: float = DEFAULT_TIMEOUT):
+        self.timeout = timeout
+        self._sessions: dict[str | None, AsyncSession] = {}
+
+    def _session(self, impersonate: str | None) -> AsyncSession:
+        if impersonate not in self._sessions:
+            self._sessions[impersonate] = AsyncSession(impersonate=impersonate)
+        return self._sessions[impersonate]
+
+    async def close(self) -> None:
+        for s in self._sessions.values():
+            await s.close()
+        self._sessions.clear()
+
+    async def get(
+        self,
+        url: str,
+        *,
+        headers: dict[str, str] | None = None,
+        cookies: dict[str, str] | None = None,
+        impersonate: str | None = "chrome",
+        params: dict | None = None,
+        timeout: float | None = None,
+        method: str = "GET",
+        json_body: dict | None = None,
+        data: dict | None = None,
+        multipart=None,
+    ) -> Fetched:
+        check_request(method, url, authenticated=bool(cookies))
+        hdrs = dict(headers or {})
+        if impersonate is None:
+            hdrs.setdefault("User-Agent", USER_AGENT)
+        kwargs = {
+            "headers": hdrs,
+            "params": params,
+            "json": json_body,
+            "data": data,
+            "multipart": multipart,
+            "timeout": timeout or self.timeout,
+        }
+        if not cookies:
+            r = await self._session(impersonate).request(
+                method, url, allow_redirects=True, max_redirects=8, **kwargs
+            )
+            return _wrap(r, url)
+
+        # Follow redirects by hand so every hop is re-checked by the privacy jail and
+        # cookies never follow a redirect onto another site.
+        async with AsyncSession(impersonate=impersonate) as s:
+            current = url
+            for _ in range(MAX_COOKIE_REDIRECTS + 1):
+                r = await s.request(
+                    method, current, cookies=cookies, allow_redirects=False, **kwargs
+                )
+                location = r.headers.get("location")
+                if r.status_code not in (301, 302, 303, 307, 308) or not location:
+                    return _wrap(r, url)
+                nxt = urljoin(current, location)
+                if registrable_domain(urlsplit(nxt).hostname) != registrable_domain(
+                    urlsplit(url).hostname
+                ):
+                    return _wrap(r, url)  # cross-site redirect: stop rather than leak cookies
+                check_request(method, nxt, authenticated=True)
+                current, kwargs["params"] = nxt, None
+            return _wrap(r, url)
+
+
+def _wrap(r, requested: str) -> Fetched:
+    headers = {k.lower(): v for k, v in r.headers.items()}
+    return Fetched(str(r.url), r.status_code, headers, r.content, requested=requested)

agent_wire/engine/stealth_browser.py

@@ -0,0 +1,90 @@
+"""Optional stage 4: render in a local headless browser and harvest cf_clearance.
+
+Needs the `stealth` extra (camoufox or playwright). The profile and the browser's own temp
+files live in WIRE_HOME, not the OS temp dir.
+"""
+
+import asyncio
+import importlib.util
+import os
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from agent_wire.engine.challenge import _HARD
+from agent_wire.engine.http_pool import Fetched
+
+NAV_TIMEOUT_MS = 30_000
+SETTLE_MS = 6_000  # extra wait for XHR-rendered pages; never blocks on long-polling sites
+CHALLENGE_WAIT_S = 20  # how long a JS challenge may take to clear itself
+
+
+def browser_env(home: Path) -> dict[str, str]:
+    """Point the browser process's temp files at the tmp folder inside WIRE_HOME."""
+    tmp = home / "tmp"
+    tmp.mkdir(parents=True, exist_ok=True)
+    return {**os.environ, "TMP": str(tmp), "TEMP": str(tmp), "TMPDIR": str(tmp)}
+
+
+class StealthBrowser:
+    def __init__(self, home: Path):
+        self.home = home
+        self.profile_dir = home / "browser-profile"
+
+    @staticmethod
+    def available() -> bool:
+        return bool(importlib.util.find_spec("camoufox") or importlib.util.find_spec("playwright"))
+
+    async def render(self, url: str) -> tuple[Fetched | None, str | None, str | None]:
+        if importlib.util.find_spec("camoufox"):
+            from camoufox.async_api import AsyncCamoufox
+
+            async with AsyncCamoufox(
+                headless=True, persistent_context=True, user_data_dir=str(self.profile_dir)
+            ) as context:
+                return await _capture(context, url)
+
+        from playwright.async_api import async_playwright
+
+        async with async_playwright() as p:
+            context = await p.chromium.launch_persistent_context(
+                str(self.profile_dir), headless=True, env=browser_env(self.home),
+                downloads_path=str(self.home / "tmp"),
+            )  # fmt: skip
+            try:
+                return await _capture(context, url)
+            finally:
+                await context.close()
+
+
+async def _still_challenged(page) -> bool:
+    html = (await page.content())[:65_536].lower().encode()
+    return any(s in html for s in _HARD)
+
+
+async def _capture(context, url: str):
+    page = await context.new_page()
+    response = await page.goto(url, wait_until="load", timeout=NAV_TIMEOUT_MS)
+    try:
+        await page.wait_for_load_state("networkidle", timeout=SETTLE_MS)
+    except Exception:  # noqa: BLE001,S110 - playwright TimeoutError; long-polling never goes idle
+        pass
+    for _ in range(CHALLENGE_WAIT_S):
+        if not await _still_challenged(page):
+            break
+        await asyncio.sleep(1)  # Cloudflare's JS challenge redirects to the page when solved
+    html = await page.content()
+    ua = await page.evaluate("navigator.userAgent")
+    host = urlsplit(url).hostname or ""
+    cf = next(
+        (
+            c["value"]
+            for c in await context.cookies()
+            if c["name"] == "cf_clearance" and host.endswith(c["domain"].lstrip("."))
+        ),
+        None,
+    )
+    status = response.status if response else 200
+    if status in (403, 503) and not await _still_challenged(page):
+        status = 200  # the challenge response itself was 403/503; the page we hold now is real
+    headers = {"content-type": "text/html; charset=utf-8"}
+    return Fetched(page.url, status, headers, html.encode(), requested=url), cf, ua