pyrepowiki-cli 0.1.0__py3-none-any.whl

repowiki/services/codewiki/client.py

@@ -0,0 +1,237 @@
+"""CodeWiki client: bootstrap + Google batchexecute RPC."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import ssl
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+
+from repowiki.services.codewiki import boq
+from repowiki.services.codewiki.wiki import Wiki, parse as parse_wiki
+
+SITE = "https://codewiki.google"
+ENDPOINT = f"{SITE}/_/BoqAngularSdlcAgentsUi/data/batchexecute"
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
+)
+CACHE_TTL_SECS = 6 * 60 * 60
+
+FALLBACK_BL = "boq_sdlc-agents-ui_20260504.02_p0"
+FALLBACK_SID = "-8491411211174446345"
+
+_WIZ_RE = re.compile(r"WIZ_global_data\s*=\s*(\{[\s\S]*?\});")
+
+
+class CodeWikiError(Exception):
+    """Base error for CodeWiki."""
+
+
+class CodeWikiConnectionError(CodeWikiError):
+    """Failed to reach codewiki.google."""
+
+
+@dataclass
+class _Bootstrap:
+    bl: str
+    sid: str
+    fetched_at: float = 0.0
+
+
+def extract_bootstrap(html: str) -> _Bootstrap:
+    match = _WIZ_RE.search(html)
+    if match is None:
+        raise ValueError("WIZ_global_data not found in page HTML")
+    try:
+        data = json.loads(match.group(1))
+    except json.JSONDecodeError as exc:
+        raise ValueError("WIZ_global_data is not valid JSON") from exc
+    bl = data.get("cfb2h")
+    sid = data.get("FdrFJe")
+    if not isinstance(bl, str):
+        raise ValueError("missing cfb2h (build label) in WIZ_global_data")
+    if not isinstance(sid, str):
+        raise ValueError("missing FdrFJe (session id) in WIZ_global_data")
+    return _Bootstrap(bl=bl, sid=sid, fetched_at=time.time())
+
+
+def _cache_path() -> Path | None:
+    if env := os.environ.get("CODEWIKI_CACHE_DIR"):
+        return Path(env) / "bootstrap.json"
+    base = os.environ.get("XDG_CACHE_HOME") or (Path.home() / ".cache")
+    return Path(base) / "codewiki" / "bootstrap.json"
+
+
+def _read_cache(path: Path) -> _Bootstrap | None:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return _Bootstrap(
+            bl=data["bl"], sid=data["sid"], fetched_at=float(data.get("fetched_at", 0))
+        )
+    except (OSError, ValueError, KeyError, TypeError):
+        return None
+
+
+def _write_cache(path: Path | None, bs: _Bootstrap) -> None:
+    if path is None:
+        return
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(
+            json.dumps({"bl": bs.bl, "sid": bs.sid, "fetched_at": bs.fetched_at}),
+            encoding="utf-8",
+        )
+    except OSError:
+        pass
+
+
+async def _fetch_bootstrap(http: httpx.AsyncClient) -> _Bootstrap:
+    resp = await http.get(SITE)
+    resp.raise_for_status()
+    return extract_bootstrap(resp.text)
+
+
+async def _load_bootstrap(http: httpx.AsyncClient) -> _Bootstrap:
+    path = _cache_path()
+    cached = _read_cache(path) if path is not None else None
+    if cached is not None and time.time() - cached.fetched_at < CACHE_TTL_SECS:
+        return cached
+    try:
+        fresh = await _fetch_bootstrap(http)
+    except Exception:
+        return _Bootstrap(bl=FALLBACK_BL, sid=FALLBACK_SID, fetched_at=time.time())
+    _write_cache(path, fresh)
+    return fresh
+
+
+class CodeWikiClient:
+    """Stateless client; the bootstrap pair is cached on disk (6h TTL)."""
+
+    def __init__(self, transport: httpx.AsyncBaseTransport | None = None) -> None:
+        self._transport = transport
+
+    def _client(self) -> httpx.AsyncClient:
+        return httpx.AsyncClient(
+            timeout=30.0,
+            verify=True,
+            headers={"User-Agent": USER_AGENT},
+            transport=self._transport,
+        )
+
+    async def read_wiki(self, repo: str) -> Wiki:
+        async with self._client() as http:
+            bs = await _load_bootstrap(http)
+            inner = json.dumps([_github_url(repo)])
+            payload = await self._call(http, bs, "VSX6ub", repo, inner)
+            return parse_wiki(payload)
+
+    async def ask(self, repo: str, question: str) -> str:
+        async with self._client() as http:
+            bs = await _load_bootstrap(http)
+            inner = json.dumps([[[question, "user"]], [None, _github_url(repo)]])
+            payload = await self._call(http, bs, "EgIxfe", repo, inner)
+            if isinstance(payload, list) and payload and isinstance(payload[0], str):
+                return payload[0]
+            raise CodeWikiError("EgIxfe response did not contain an answer string")
+
+    async def github_head(self, repo: str) -> dict:
+        """Return the GitHub HEAD sha and commit date for ``repo``.
+
+        Used to compare the wiki's ``commit_sha`` against the live branch to
+        detect a stale wiki.
+        """
+        owner, name = repo.split("/", 1)
+        try:
+            async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+                resp = await client.get(
+                    f"https://api.github.com/repos/{owner}/{name}/commits/HEAD"
+                )
+                resp.raise_for_status()
+                commit = resp.json()
+        except httpx.TransportError as exc:
+            message = f"Failed to connect to GitHub: {exc}"
+            if _is_cert_error(exc):
+                message += (
+                    " (TLS certificate verification failed; set SSL_CERT_FILE to "
+                    "your CA bundle to trust a proxy/mirror)"
+                )
+            raise CodeWikiConnectionError(message) from exc
+        except httpx.HTTPStatusError as exc:
+            raise CodeWikiError(
+                f"GitHub API returned HTTP {exc.response.status_code}"
+            ) from exc
+        if not isinstance(commit, dict):
+            raise CodeWikiError("unexpected GitHub response")
+        committer = commit.get("commit", {}).get("committer") or {}
+        when = committer.get("date") if isinstance(committer, dict) else None
+        return {"sha": commit.get("sha") or "", "when": when}
+
+    async def _call(
+        self,
+        http: httpx.AsyncClient,
+        bs: _Bootstrap,
+        rpc_id: str,
+        repo: str,
+        inner_json: str,
+    ) -> object:
+        params = {
+            "rpcids": rpc_id,
+            "source-path": f"/github.com/{repo}",
+            "bl": bs.bl,
+            "f.sid": bs.sid,
+            "hl": "en-US",
+            "_reqid": _pseudo_reqid(),
+            "rt": "c",
+        }
+        headers = {
+            "Accept-Language": "en-US,en;q=0.9",
+            "Origin": SITE,
+            "Referer": f"{SITE}/github.com/{repo}",
+            "X-Same-Domain": "1",
+            "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
+        }
+        try:
+            resp = await http.post(
+                ENDPOINT,
+                params=params,
+                headers=headers,
+                content=boq.encode_request(rpc_id, inner_json),
+            )
+        except httpx.TransportError as exc:
+            message = f"Failed to connect to CodeWiki: {exc}"
+            if _is_cert_error(exc):
+                message += (
+                    " (TLS certificate verification failed; set SSL_CERT_FILE to "
+                    "your CA bundle to trust a proxy/mirror)"
+                )
+            raise CodeWikiConnectionError(message) from exc
+        if resp.status_code != 200:
+            raise CodeWikiError(
+                f"batchexecute returned HTTP {resp.status_code}: {resp.text[:200]}"
+            )
+        try:
+            return boq.decode_response(resp.text, rpc_id)
+        except ValueError as exc:
+            raise CodeWikiError(str(exc)) from exc
+
+
+def _github_url(repo: str) -> str:
+    return f"https://github.com/{repo}"
+
+
+def _pseudo_reqid() -> int:
+    return 100_000 + (int(time.time()) % 900_000)
+
+
+def _is_cert_error(exc: BaseException) -> bool:
+    while exc is not None:
+        if isinstance(exc, ssl.SSLCertVerificationError):
+            return True
+        exc = exc.__cause__ or exc.__context__
+    return False

repowiki/services/codewiki/wiki.py

@@ -0,0 +1,162 @@
+"""Parse and render CodeWiki VSX6ub wiki payloads."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from urllib.parse import unquote
+
+_GITHUB_PREFIX = "https://github.com"
+
+
+@dataclass
+class Section:
+    title: str
+    level: int
+    markdown: str
+    diagrams: list[str] = field(default_factory=list)
+
+
+@dataclass
+class Wiki:
+    repo_slug: str
+    commit_sha: str
+    sections: list[Section]
+
+
+def parse(payload: object) -> Wiki:
+    """Parse a VSX6ub payload: ``[wiki, [null, url], true, n]``."""
+    if not isinstance(payload, list) or not payload:
+        raise ValueError("missing wiki container")
+    container = payload[0]
+    if not isinstance(container, list) or not container:
+        raise ValueError("missing wiki container")
+    header = container[0]
+    if not isinstance(header, list) or not header:
+        raise ValueError("missing wiki header array")
+    repo_slug = header[0] if header and isinstance(header[0], str) else ""
+    if not repo_slug:
+        raise ValueError("missing repo slug")
+    commit_sha = header[1] if len(header) > 1 and isinstance(header[1], str) else ""
+    raw_sections = container[1] if len(container) > 1 else None
+    if not isinstance(raw_sections, list):
+        raise ValueError("missing sections array")
+    sections = [parse_section(s, i) for i, s in enumerate(raw_sections)]
+    return Wiki(repo_slug=repo_slug, commit_sha=commit_sha, sections=sections)
+
+
+def parse_section(value: object, index: int) -> Section:
+    if not isinstance(value, list):
+        raise ValueError(f"section #{index} is not an array")
+    title = value[0] if value and isinstance(value[0], str) else ""
+    if not title:
+        raise ValueError(f"missing title in section #{index}")
+    level = value[1] if len(value) > 1 and isinstance(value[1], int) else 1
+    markdown = ""
+    if len(value) > 5 and isinstance(value[5], str):
+        markdown = value[5]
+    elif len(value) > 4 and isinstance(value[4], str):
+        markdown = value[4]
+    diagrams = extract_diagrams(value[7] if len(value) > 7 else None)
+    return Section(title=title, level=level, markdown=markdown, diagrams=diagrams)
+
+
+def extract_diagrams(outer: object) -> list[str]:
+    out: list[str] = []
+    if not isinstance(outer, list):
+        return out
+    for group in outer:
+        if not isinstance(group, list):
+            continue
+        for diagram in group:
+            if not isinstance(diagram, list):
+                continue
+            if len(diagram) > 4 and isinstance(diagram[4], str) and diagram[4]:
+                out.append(diagram[4])
+    return out
+
+
+def render_structure(wiki: Wiki) -> str:
+    lines = []
+    for s in wiki.sections:
+        indent = "  " * max(0, s.level - 1)
+        lines.append(f"{indent}- {s.title}")
+    return "\n".join(lines) + ("\n" if lines else "")
+
+
+def render_section(s: Section) -> str:
+    """Render a single section as Markdown (heading + body + dot diagrams)."""
+    hashes = "#" * min(max(s.level, 1), 6)
+    parts = [f"{hashes} {s.title}\n"]
+    body = resolve_links(s.markdown)
+    parts.append(body)
+    if body and not body.endswith("\n"):
+        parts.append("\n")
+    for dot in s.diagrams:
+        parts.append(f"\n```dot\n{dot.strip()}\n```\n")
+    parts.append("\n")
+    return "".join(parts)
+
+
+def render_markdown(wiki: Wiki) -> str:
+    parts = [f"# {wiki.repo_slug} (commit {wiki.commit_sha})\n"]
+    for s in wiki.sections:
+        parts.append(render_section(s))
+    return "".join(parts)
+
+
+def render_page(wiki: Wiki, title: str) -> str:
+    """Render the single section whose title matches (case-insensitive exact).
+
+    Raises ValueError listing available titles if none matches.
+    """
+    target = title.strip().lower()
+    for s in wiki.sections:
+        if s.title.strip().lower() == target:
+            return render_section(s).strip()
+    available = "\n".join(f"  - {s.title}" for s in wiki.sections) or "  - (none)"
+    raise ValueError(f"Page '{title}' not found. Available pages:\n{available}")
+
+
+def resolve_links(markdown: str) -> str:
+    """Rewrite ``](%2F...)`` / ``](/...)`` targets to ``](https://github.com/...)``."""
+    out: list[str] = []
+    rest = markdown
+    while True:
+        open_idx = rest.find("](")
+        if open_idx == -1:
+            out.append(rest)
+            break
+        out.append(rest[:open_idx])
+        after = rest[open_idx + 2:]
+        end = _find_link_end(after)
+        if end is None:
+            out.append("](")
+            rest = after
+            continue
+        out.append("](")
+        out.append(_rewrite_target(after[:end]))
+        out.append(")")
+        rest = after[end + 1:]
+    return "".join(out)
+
+
+def _find_link_end(s: str) -> int | None:
+    depth = 0
+    for i, ch in enumerate(s):
+        if ch == "(":
+            depth += 1
+        elif ch == ")":
+            if depth == 0:
+                return i
+            depth -= 1
+    return None
+
+
+def _rewrite_target(target: str) -> str:
+    needs_rewrite = target.startswith(("%2F", "%2f", "/"))
+    if not needs_rewrite:
+        return target
+    decoded = unquote(target)
+    if decoded.startswith("/"):
+        return f"{_GITHUB_PREFIX}{decoded}"
+    return decoded

repowiki/services/deepwiki/__init__.py

@@ -0,0 +1,5 @@
+"""DeepWiki service."""
+
+from repowiki.services.deepwiki.cli import register
+
+__all__ = ["register"]