agent-wire-cli 0.2.1__py3-none-any.whl

agent_wire/cache/sqlite_ledger.py

@@ -0,0 +1,132 @@
+"""L2 on-disk cache: per-domain winning waterfall stage + 500-row circular HTTP cache."""
+
+import hashlib
+import sqlite3
+import time
+from pathlib import Path
+
+import zstandard
+
+ROUTE_TTL_S = 24 * 3600
+
+# rowid breaks updated_at ties (many inserts within one second) so the oldest row goes first.
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS domain_routes (
+    domain TEXT PRIMARY KEY,
+    winning_stage INTEGER NOT NULL,
+    cf_clearance TEXT,
+    user_agent TEXT,
+    updated_at INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS http_etag_cache (
+    url_hash TEXT PRIMARY KEY,
+    url TEXT NOT NULL,
+    etag TEXT,
+    last_modified TEXT,
+    content_hash TEXT NOT NULL,
+    compressed_body BLOB,
+    updated_at INTEGER NOT NULL
+);
+CREATE TRIGGER IF NOT EXISTS limit_etag_cache_size
+AFTER INSERT ON http_etag_cache
+WHEN (SELECT COUNT(*) FROM http_etag_cache) > 500
+BEGIN
+    DELETE FROM http_etag_cache
+    WHERE url_hash IN (
+        SELECT url_hash FROM http_etag_cache
+        ORDER BY updated_at ASC, rowid ASC
+        LIMIT (SELECT COUNT(*) - 500 FROM http_etag_cache)
+    );
+END;
+"""
+
+
+class Ledger:
+    def __init__(self, path: Path):
+        self._db = sqlite3.connect(path)
+        self._db.execute("PRAGMA journal_mode=WAL")
+        self._db.execute("PRAGMA synchronous=NORMAL")
+        self._db.executescript(SCHEMA)
+        self._zc = zstandard.ZstdCompressor(level=3)
+        self._zd = zstandard.ZstdDecompressor()
+
+    def close(self) -> None:
+        self._db.close()
+
+    def get_winning_stage(self, domain: str, ttl_s: int = ROUTE_TTL_S) -> int | None:
+        row = self._db.execute(
+            "SELECT winning_stage FROM domain_routes WHERE domain = ? AND updated_at > ?",
+            (domain, int(time.time()) - ttl_s),
+        ).fetchone()
+        return row[0] if row else None
+
+    def get_route(self, domain: str, ttl_s: int = ROUTE_TTL_S) -> dict | None:
+        row = self._db.execute(
+            "SELECT winning_stage, cf_clearance, user_agent FROM domain_routes"
+            " WHERE domain = ? AND updated_at > ?",
+            (domain, int(time.time()) - ttl_s),
+        ).fetchone()
+        if row is None:
+            return None
+        return {"stage": row[0], "cf_clearance": row[1], "user_agent": row[2]}
+
+    def routes(self) -> list[tuple[str, int]]:
+        return self._db.execute(
+            "SELECT domain, winning_stage FROM domain_routes ORDER BY updated_at DESC"
+        ).fetchall()
+
+    def clear_route(self, domain: str) -> None:
+        with self._db:
+            self._db.execute("DELETE FROM domain_routes WHERE domain = ?", (domain,))
+
+    def set_winning_stage(
+        self,
+        domain: str,
+        stage: int,
+        cf_clearance: str | None = None,
+        user_agent: str | None = None,
+    ) -> None:
+        with self._db:
+            self._db.execute(
+                "INSERT OR REPLACE INTO domain_routes VALUES (?, ?, ?, ?, ?)",
+                (domain, stage, cf_clearance, user_agent, int(time.time())),
+            )
+
+    def get_cached(self, url: str) -> dict | None:
+        row = self._db.execute(
+            "SELECT etag, last_modified, content_hash, compressed_body"
+            " FROM http_etag_cache WHERE url_hash = ?",
+            (_url_hash(url),),
+        ).fetchone()
+        if row is None:
+            return None
+        etag, last_modified, content_hash, blob = row
+        body = self._zd.decompress(blob) if blob is not None else None
+        return {
+            "etag": etag,
+            "last_modified": last_modified,
+            "content_hash": content_hash,
+            "body": body,
+        }
+
+    def put_cached(
+        self,
+        url: str,
+        content_hash: str,
+        body: bytes | None = None,
+        etag: str | None = None,
+        last_modified: str | None = None,
+    ) -> None:
+        blob = self._zc.compress(body) if body is not None else None
+        with self._db:
+            self._db.execute(
+                "INSERT OR REPLACE INTO http_etag_cache VALUES (?, ?, ?, ?, ?, ?, ?)",
+                (_url_hash(url), url, etag, last_modified, content_hash, blob, int(time.time())),
+            )
+
+    def cache_rows(self) -> int:
+        return self._db.execute("SELECT COUNT(*) FROM http_etag_cache").fetchone()[0]
+
+
+def _url_hash(url: str) -> str:
+    return hashlib.sha256(url.encode()).hexdigest()

agent_wire/channels/__init__.py

@@ -0,0 +1,41 @@
+"""Lazy channel registry: modules import on first use so `wire serve` starts fast."""
+
+import importlib
+
+MODULES = {
+    "web": "tier1_global.web",
+    "search": "tier1_global.search",
+    "github": "tier1_global.github",
+    "reddit": "tier1_global.reddit",
+    "x": "tier1_global.x_twitter",
+    "youtube": "tier1_global.youtube",
+    "hackernews": "tier1_global.hackernews",
+    "stackoverflow": "tier1_global.stackoverflow",
+    "wikipedia": "tier1_global.wikipedia",
+    "rss": "tier1_global.rss",
+    "arxiv": "tier2_pro.arxiv_papers",
+    "linkedin": "tier2_pro.linkedin",
+    "hubs": "tier2_pro.hubs",
+    "markets": "tier2_pro.markets",
+    "meta_social": "tier2_pro.meta_social",
+    "publishing": "tier2_pro.publishing",
+    "fediverse": "tier2_pro.fediverse",
+    "archive": "tier2_pro.archive_bypass",
+    "polymarket": "tier2_pro.polymarket",
+    "news": "tier2_pro.news",
+    "stocktwits": "tier2_pro.stocktwits",
+    "devto": "tier2_pro.devto",
+    "bilibili": "tier3_regional.bilibili",
+    "xiaohongshu": "tier3_regional.xiaohongshu",
+    "wechat": "tier3_regional.wechat",
+    "zhihu": "tier3_regional.zhihu",
+    "v2ex": "tier3_regional.v2ex",
+    "douyin_tiktok": "tier3_regional.douyin_tiktok",
+    "xiaoyuzhou": "tier3_regional.xiaoyuzhou",
+    "xueqiu": "tier3_regional.xueqiu",
+    "boss_zhipin": "tier3_regional.boss_zhipin",
+}
+
+
+def channel_class(name: str):
+    return importlib.import_module(f"agent_wire.channels.{MODULES[name]}").CHANNEL

agent_wire/channels/base.py

@@ -0,0 +1,249 @@
+"""Channel interface, shared context and helpers used by every platform module."""
+
+import html as htmllib
+import json
+import re
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+from typing import ClassVar
+
+from selectolax.lexbor import LexborHTMLParser
+
+from agent_wire.engine.http_pool import Fetched, HttpPool
+from agent_wire.engine.waterfall import Waterfall
+from agent_wire.firewall.inbound_shield import clean_html
+from agent_wire.firewall.outbound_privacy import parse_json
+from agent_wire.models import WireDocument, WireSearchHit
+
+
+class WireUnsupported(Exception):
+    """The channel can't do this (yet, or without a login); the message says what to do."""
+
+
+@dataclass
+class Context:
+    pool: HttpPool
+    waterfall: Waterfall
+    vault: object | None = None
+    ledger: object | None = None
+    _channels: dict = field(default_factory=dict)
+
+    def channel(self, name: str) -> "Channel":
+        if name not in self._channels:
+            from agent_wire.channels import channel_class
+
+            self._channels[name] = channel_class(name)(self)
+        return self._channels[name]
+
+
+class Channel:
+    name: ClassVar[str] = ""
+
+    def __init__(self, ctx: Context):
+        self.ctx = ctx
+
+    async def read(
+        self, url: str, *, query: str | None = None, section: str | None = None,
+        include_comments: bool = True,
+    ) -> WireDocument:  # fmt: skip
+        raise WireUnsupported(f"{self.name} can't read URLs")
+
+    async def search(
+        self, query: str, *, limit: int = 5, sort: str = "relevance"
+    ) -> list[WireSearchHit]:
+        raise WireUnsupported(f"{self.name} has no search; try platforms=['web']")
+
+    async def feed(self, target: str, *, sort: str = "hot", limit: int = 10) -> WireDocument:
+        raise WireUnsupported(f"{self.name} has no feeds")
+
+    async def me(self, target: str = "latest_posts", *, limit: int = 5) -> WireDocument:
+        raise WireUnsupported(f"{self.name} has no account inspection")
+
+    # ---- helpers -----------------------------------------------------------------------------
+    async def fetch(self, url: str, **kw) -> Fetched:
+        kw.setdefault("channel", self.name)
+        return await self.ctx.waterfall.fetch(url, **kw)
+
+    async def fetch_json(self, url: str, **kw):
+        """API call: no archive/browser stages; identity keys stripped (outbound layer 3)."""
+        kw.setdefault("archive", False)
+        kw.setdefault("browser", False)
+        f = await self.fetch(url, **kw)
+        return parse_json(f.content), f
+
+    def doc(self, f: Fetched | None, url: str, **fields) -> WireDocument:
+        if f is not None:
+            fields.setdefault("stage", f.stage)
+            fields.setdefault("auth", f.auth)
+            fields.setdefault("raw_bytes", len(f.content))
+        return WireDocument(url=url, channel=self.name, **fields)
+
+
+def ts(value) -> datetime | None:
+    """Epoch seconds/ms or ISO-8601 -> aware datetime."""
+    if value in (None, ""):
+        return None
+    try:
+        if isinstance(value, int | float) or str(value).isdigit():
+            v = float(value)
+            return datetime.fromtimestamp(v / 1000 if v > 1e11 else v, UTC)
+        return datetime.fromisoformat(str(value))
+    except (ValueError, OSError, OverflowError):
+        return None
+
+
+_REL = re.compile(r"(\d+)\s*(second|minute|hour|day|week|month|year)s?\s+ago", re.IGNORECASE)
+_REL_S = {"second": 1, "minute": 60, "hour": 3600, "day": 86400, "week": 7 * 86400,
+          "month": 30 * 86400, "year": 365 * 86400}  # fmt: skip
+
+
+def relative_ts(text: str | None, now: datetime | None = None) -> datetime | None:
+    """'3 weeks ago' / 'Streamed 2 days ago' -> approximate aware datetime."""
+    m = _REL.search(text or "")
+    if not m:
+        return None
+    return (now or datetime.now(UTC)) - timedelta(seconds=int(m[1]) * _REL_S[m[2].lower()])
+
+
+def listing(items: list[dict]) -> str:
+    """Compact DAM list for feeds/search: one line per item plus an optional indented snippet."""
+    lines = []
+    for i, it in enumerate(items, 1):
+        bits = [b for b in (it.get("meta") or ()) if b not in (None, "")]
+        title = " ".join(str(it.get("title") or it.get("url")).split())  # newlines break the link
+        head = f"{i}. [{title}]({it.get('url')})"
+        lines.append(head + (f" · {' · '.join(map(str, bits))}" if bits else ""))
+        if snippet := (it.get("snippet") or "").strip():
+            lines.append("   " + " ".join(snippet.split())[:280])
+    return "\n".join(lines)
+
+
+def compact_count(text: str | int | None) -> int:
+    """'1.2K' / '3,400' / '2M likes' -> int."""
+    if text is None:
+        return 0
+    if isinstance(text, int):
+        return text
+    m = re.search(r"([\d.,]+)\s*([KMB万亿])?", str(text), re.IGNORECASE)
+    if not m:
+        return 0
+    num = float(m[1].replace(",", "")) if m[1].replace(",", "").replace(".", "").isdigit() else 0
+    mult = {"K": 1e3, "M": 1e6, "B": 1e9, "万": 1e4, "亿": 1e8}.get((m[2] or "").upper(), 1)
+    return int(num * mult)
+
+
+# ---- HTML -> compact Markdown -------------------------------------------------------------------
+_BLOCK = frozenset(
+    {"p", "div", "section", "article", "main", "header", "footer", "figure", "details", "summary",
+     "dl", "dd", "dt", "tr", "table", "tbody", "thead"}
+)  # fmt: skip
+_SKIP = frozenset({"img", "svg", "picture", "video", "audio", "iframe", "button", "form", "nav"})
+_WS = re.compile(r"[ \t\r\f\v]+")
+_BLANKS = re.compile(r"\n{3,}")
+
+
+def html_to_markdown(html: str, *, links: bool = True) -> str:
+    """Untrusted HTML fragment -> Markdown, after the inbound shield removed hidden nodes."""
+    if not html:
+        return ""
+    tree = LexborHTMLParser(clean_html(html))
+    root = tree.body or tree.root
+    out = _md(root, links) if root else ""
+    return _BLANKS.sub("\n\n", "\n".join(ln.rstrip() for ln in out.splitlines())).strip()
+
+
+def _md(node, links: bool) -> str:
+    parts = []
+    for child in node.iter(include_text=True):
+        tag = child.tag
+        if tag == "-text":
+            parts.append(
+                _WS.sub(" ", htmllib.unescape(child.text_content or "")).replace("\n", " ")
+            )
+        elif tag in _SKIP or tag.startswith("-"):
+            continue
+        elif tag == "pre":
+            parts.append(f"\n\n```\n{child.text(deep=True).strip(chr(10))}\n```\n\n")
+        elif tag == "code":
+            parts.append(f"`{child.text(deep=True)}`")
+        elif tag == "br":
+            parts.append("\n")
+        elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            parts.append(f"\n\n{'#' * int(tag[1])} {_md(child, links).strip()}\n\n")
+        elif tag == "li":
+            parts.append(f"\n- {_md(child, links).strip()}")
+        elif tag in ("ul", "ol"):
+            parts.append(f"\n{_md(child, links)}\n")
+        elif tag == "blockquote":
+            inner = _md(child, links).strip()
+            parts.append("\n\n" + "\n".join(f"> {ln}" for ln in inner.splitlines()) + "\n\n")
+        elif tag == "a":
+            text = _md(child, links).strip()
+            href = child.attributes.get("href") or ""
+            use = links and href.startswith("http") and text and text != href
+            parts.append(f"[{text}]({href})" if use else text or href)
+        elif tag in ("td", "th"):
+            parts.append(f" {_md(child, links).strip()} |")
+        elif tag == "tr":
+            parts.append(f"\n|{_md(child, links)}")
+        elif tag in _BLOCK:
+            parts.append(f"\n\n{_md(child, links).strip()}\n\n")
+        else:
+            parts.append(_md(child, links))
+    return "".join(parts)
+
+
+def html_text(html: str) -> str:
+    return " ".join(LexborHTMLParser(clean_html(html)).text(separator=" ").split()) if html else ""
+
+
+# ---- structured data embedded in pages ----------------------------------------------------------
+_JSON = json.JSONDecoder()
+
+
+def meta_tags(html: str) -> dict[str, str]:
+    """og:/twitter:/name= meta tags, lower-cased keys."""
+    out = {}
+    for m in LexborHTMLParser(html).css("meta[property], meta[name]"):
+        key = (m.attributes.get("property") or m.attributes.get("name") or "").lower()
+        if key and (content := m.attributes.get("content")):
+            out.setdefault(key, htmllib.unescape(content))
+    return out
+
+
+def json_ld(html: str) -> list[dict]:
+    items = []
+    for node in LexborHTMLParser(html).css('script[type="application/ld+json"]'):
+        try:
+            data = json.loads(node.text())
+        except ValueError:
+            continue
+        for d in data if isinstance(data, list) else [data]:
+            if isinstance(d, dict):
+                items += d["@graph"] if isinstance(d.get("@graph"), list) else [d]
+    return items
+
+
+def script_json(html: str, selector: str):
+    """JSON inside a <script> matched by CSS selector (e.g. script#__NEXT_DATA__)."""
+    node = LexborHTMLParser(html).css_first(selector)
+    if not node:
+        return None
+    try:
+        return json.loads(node.text())
+    except ValueError:
+        return None
+
+
+def assigned_json(html: str, var: str, *, js_undefined: bool = False):
+    """The object literal assigned to `var` in inline JS (`window.__INITIAL_STATE__ = {...}`)."""
+    m = re.search(rf"{re.escape(var)}\s*=\s*", html)
+    if not m:
+        return None
+    text = html[m.end() :]
+    if js_undefined:  # JS-only literal some sites leave in their state dumps
+        text = re.sub(r"([:\[,]\s*)undefined(?=\s*[,}\]])", r"\1null", text)
+    try:
+        return _JSON.raw_decode(text)[0]
+    except ValueError:
+        return None