cli-web-codewiki 0.1.0__py3-none-any.whl

cli_web/codewiki/core/client.py

@@ -0,0 +1,224 @@
+"""HTTP client for Code Wiki batchexecute API."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+import httpx
+
+from .exceptions import (
+    AuthError,
+    CodeWikiError,
+    NetworkError,
+    NotFoundError,
+    RateLimitError,
+    ServerError,
+)
+from .models import ChatResponse, Repository, WikiPage, WikiSection
+from .rpc.decoder import decode_response
+from .rpc.encoder import build_url, encode_request
+from .rpc.types import DEFAULT_HEADERS, RPCMethod
+
+
+class CodeWikiClient:
+    """Client for Google Code Wiki batchexecute API."""
+
+    def __init__(self) -> None:
+        self._http = httpx.Client(
+            headers=DEFAULT_HEADERS,
+            timeout=30.0,
+            follow_redirects=True,
+        )
+
+    def close(self) -> None:
+        self._http.close()
+
+    def __enter__(self) -> CodeWikiClient:
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+    def _call(self, rpc_id: str, params: list) -> Any:
+        """Make a batchexecute RPC call."""
+        url = build_url(rpc_id)
+        body = encode_request(rpc_id, params)
+
+        try:
+            resp = self._http.post(
+                url,
+                content=body.encode("utf-8"),
+            )
+        except httpx.ConnectError as exc:
+            raise NetworkError(f"Connection failed: {exc}") from exc
+        except httpx.TimeoutException as exc:
+            raise NetworkError(f"Request timed out: {exc}") from exc
+
+        if resp.status_code in (401, 403):
+            raise AuthError("Authentication required", recoverable=False)
+        if resp.status_code == 404:
+            raise NotFoundError("Resource not found")
+        if resp.status_code == 429:
+            retry = resp.headers.get("Retry-After")
+            raise RateLimitError(
+                "Rate limited by Code Wiki",
+                retry_after=float(retry) if retry else None,
+            )
+        if resp.status_code >= 500:
+            raise ServerError(f"Server error: {resp.status_code}", status_code=resp.status_code)
+        if resp.status_code >= 400:
+            raise CodeWikiError(f"HTTP {resp.status_code}: {resp.text[:200]}")
+
+        return decode_response(resp.content, rpc_id)
+
+    # ── Repos ───────────────────────────────────────────────
+
+    def featured_repos(self) -> list[Repository]:
+        """Get featured repositories from the homepage."""
+        raw = self._call(RPCMethod.FEATURED_REPOS, [])
+        if not raw:
+            return []
+        # Response is [[repo1, repo2, ...]] — unwrap outer array
+        items = (
+            raw[0] if isinstance(raw[0], list) and raw[0] and isinstance(raw[0][0], list) else raw
+        )
+        repos = []
+        for item in items:
+            if not isinstance(item, list) or len(item) < 6:
+                continue
+            slug = item[0] or ""
+            meta = item[3] if len(item) > 3 and isinstance(item[3], list) else [None, ""]
+            info = item[5] if len(item) > 5 and isinstance(item[5], list) else []
+            repos.append(
+                Repository(
+                    slug=slug,
+                    github_url=meta[1] if len(meta) > 1 else "",
+                    description=info[0] if len(info) > 0 else "",
+                    avatar_url=info[1] if len(info) > 1 else "",
+                    stars=info[2] if len(info) > 2 else 0,
+                )
+            )
+        return repos
+
+    def search_repos(self, query: str, limit: int = 25, offset: int = 0) -> list[Repository]:
+        """Search for repositories."""
+        raw = self._call(RPCMethod.SEARCH_REPOS, [query, limit, query, offset])
+        if not raw:
+            return []
+        # Response is [[repo1, repo2, ...]] — unwrap outer array
+        items = (
+            raw[0] if isinstance(raw[0], list) and raw[0] and isinstance(raw[0][0], list) else raw
+        )
+        repos = []
+        for item in items:
+            if not isinstance(item, list) or len(item) < 6:
+                continue
+            slug = item[0] or ""
+            meta = item[3] if isinstance(item[3], list) else [None, ""]
+            ts = item[4] if isinstance(item[4], list) else [None, None]
+            info = item[5] if isinstance(item[5], list) else []
+            updated = None
+            if ts and ts[0]:
+                try:
+                    updated = datetime.fromtimestamp(ts[0])
+                except (ValueError, OSError, TypeError):
+                    pass
+            repos.append(
+                Repository(
+                    slug=slug,
+                    github_url=meta[1] if len(meta) > 1 else "",
+                    description=info[0] if len(info) > 0 else "",
+                    avatar_url=info[1] if len(info) > 1 else "",
+                    stars=info[2] if len(info) > 2 else 0,
+                    updated_at=updated,
+                )
+            )
+        return repos
+
+    # ── Wiki ────────────────────────────────────────────────
+
+    def get_wiki(self, repo_slug: str) -> WikiPage:
+        """Get the full wiki page for a repository."""
+        github_url = f"https://github.com/{repo_slug}"
+        raw = self._call(RPCMethod.WIKI_PAGE, [github_url])
+        if not raw:
+            raise NotFoundError(f"Wiki not found for {repo_slug}")
+
+        top = raw[0] if isinstance(raw, list) and raw else []
+        if not isinstance(top, list) or len(top) < 2:
+            raise NotFoundError(f"Wiki not found for {repo_slug}")
+
+        repo_info = top[0] if isinstance(top[0], list) else []
+        sections_raw = top[1] if isinstance(top[1], list) else []
+        ts = top[4] if len(top) > 4 and isinstance(top[4], list) else [None, None]
+
+        commit_hash = repo_info[1] if len(repo_info) > 1 else ""
+        updated = None
+        if ts and ts[0]:
+            try:
+                updated = datetime.fromtimestamp(ts[0])
+            except (ValueError, OSError, TypeError):
+                pass
+
+        meta = raw[1] if len(raw) > 1 and isinstance(raw[1], list) else []
+        has_wiki = meta[1] if len(meta) > 1 else False
+
+        sections = []
+        for sec in sections_raw:
+            if not isinstance(sec, list) or len(sec) < 2:
+                continue
+            code_refs = []
+            if len(sec) > 3 and isinstance(sec[3], list):
+                for ref in sec[3]:
+                    if isinstance(ref, list) and ref:
+                        code_refs.append(ref[0])
+            sections.append(
+                WikiSection(
+                    title=sec[0] or "",
+                    level=sec[1] if len(sec) > 1 else 1,
+                    description=sec[2] if len(sec) > 2 else "",
+                    code_refs=code_refs,
+                    content=sec[4] if len(sec) > 4 else "",
+                )
+            )
+
+        repo = Repository(
+            slug=repo_slug,
+            github_url=github_url,
+            commit_hash=commit_hash,
+            updated_at=updated,
+        )
+
+        return WikiPage(repo=repo, sections=sections, has_wiki=has_wiki)
+
+    # ── Chat ────────────────────────────────────────────────
+
+    def chat(
+        self,
+        question: str,
+        repo_slug: str,
+        history: list[tuple[str, str]] | None = None,
+    ) -> ChatResponse:
+        """Ask Gemini a question about a repository.
+
+        Args:
+            question: The user's question.
+            repo_slug: Repository slug (e.g. "excalidraw/excalidraw").
+            history: Optional conversation history as [(text, role), ...].
+        """
+        messages = []
+        if history:
+            for text, role in history:
+                messages.append([text, role])
+        messages.append([question, "user"])
+
+        github_url = f"https://github.com/{repo_slug}"
+        params = [messages, [None, github_url]]
+
+        raw = self._call(RPCMethod.CHAT, params)
+        if not raw:
+            raise NotFoundError(f"No response for {repo_slug}")
+
+        answer = raw[0] if isinstance(raw, list) and raw else str(raw)
+        return ChatResponse(answer=answer, repo_slug=repo_slug)

cli_web/codewiki/core/exceptions.py

@@ -0,0 +1,74 @@
+"""Domain-specific exception hierarchy for cli-web-codewiki."""
+
+from __future__ import annotations
+
+
+class CodeWikiError(Exception):
+    """Base exception for all Code Wiki errors."""
+
+    def to_dict(self) -> dict:
+        return {"error": True, "code": error_code_for(self), "message": str(self)}
+
+
+class AuthError(CodeWikiError):
+    """Authentication failure (unlikely for public API, but kept for completeness)."""
+
+    def __init__(self, message: str, recoverable: bool = True):
+        self.recoverable = recoverable
+        super().__init__(message)
+
+
+class RateLimitError(CodeWikiError):
+    """Rate limited by the server."""
+
+    def __init__(self, message: str, retry_after: float | None = None):
+        self.retry_after = retry_after
+        super().__init__(message)
+
+    def to_dict(self) -> dict:
+        d = super().to_dict()
+        if self.retry_after is not None:
+            d["retry_after"] = self.retry_after
+        return d
+
+
+class NetworkError(CodeWikiError):
+    """Network connectivity failure."""
+
+
+class ServerError(CodeWikiError):
+    """Server returned 5xx."""
+
+    def __init__(self, message: str, status_code: int = 500):
+        self.status_code = status_code
+        super().__init__(message)
+
+
+class NotFoundError(CodeWikiError):
+    """Requested resource not found (404 or empty RPC result)."""
+
+
+class RPCError(CodeWikiError):
+    """Batchexecute RPC-level error."""
+
+    def __init__(self, message: str, code: int | None = None):
+        self.code = code
+        super().__init__(message)
+
+
+EXCEPTION_CODE_MAP: dict[type, str] = {
+    AuthError: "AUTH_EXPIRED",
+    RateLimitError: "RATE_LIMITED",
+    NotFoundError: "NOT_FOUND",
+    ServerError: "SERVER_ERROR",
+    NetworkError: "NETWORK_ERROR",
+    RPCError: "RPC_ERROR",
+}
+
+
+def error_code_for(exc: Exception) -> str:
+    """Return standardized error code string for JSON output."""
+    for cls, code in EXCEPTION_CODE_MAP.items():
+        if isinstance(exc, cls):
+            return code
+    return "INTERNAL_ERROR"

cli_web/codewiki/core/models.py

@@ -0,0 +1,91 @@
+"""Data models for Code Wiki CLI."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+
+
+@dataclass
+class Repository:
+    """A GitHub repository on Code Wiki."""
+
+    slug: str
+    github_url: str
+    description: str = ""
+    avatar_url: str = ""
+    stars: int = 0
+    commit_hash: str = ""
+    updated_at: datetime | None = None
+
+    @property
+    def org(self) -> str:
+        parts = self.slug.split("/")
+        return parts[0] if len(parts) >= 2 else ""
+
+    @property
+    def name(self) -> str:
+        parts = self.slug.split("/")
+        return parts[1] if len(parts) >= 2 else self.slug
+
+    def to_dict(self) -> dict:
+        return {
+            "slug": self.slug,
+            "github_url": self.github_url,
+            "description": self.description,
+            "avatar_url": self.avatar_url,
+            "stars": self.stars,
+            "commit_hash": self.commit_hash,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+        }
+
+
+@dataclass
+class WikiSection:
+    """A section in a Code Wiki page."""
+
+    title: str
+    level: int
+    description: str = ""
+    code_refs: list[str] = field(default_factory=list)
+    content: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "title": self.title,
+            "level": self.level,
+            "description": self.description,
+            "code_refs": self.code_refs,
+            "content": self.content,
+        }
+
+
+@dataclass
+class WikiPage:
+    """A full Code Wiki page for a repository."""
+
+    repo: Repository
+    sections: list[WikiSection] = field(default_factory=list)
+    has_wiki: bool = False
+
+    def to_dict(self) -> dict:
+        return {
+            "repo": self.repo.to_dict(),
+            "sections": [s.to_dict() for s in self.sections],
+            "section_count": len(self.sections),
+            "has_wiki": self.has_wiki,
+        }
+
+
+@dataclass
+class ChatResponse:
+    """A Gemini chat response."""
+
+    answer: str
+    repo_slug: str
+
+    def to_dict(self) -> dict:
+        return {
+            "answer": self.answer,
+            "repo": self.repo_slug,
+        }

cli_web/codewiki/core/rpc/decoder.py

@@ -0,0 +1,86 @@
+"""Decode batchexecute RPC responses from Code Wiki."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ..exceptions import RPCError
+
+
+def _strip_prefix(raw: bytes | str) -> str:
+    """Remove the )]}' anti-XSSI prefix."""
+    text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
+    if text.startswith(")]}'"):
+        text = text[4:]
+    return text.lstrip("\n")
+
+
+def _parse_chunks(text: str) -> list[list]:
+    """Extract JSON array chunks from the batchexecute response.
+
+    The response interleaves numeric length hints with JSON arrays.
+    We use raw_decode to reliably find array boundaries.
+    """
+    decoder = json.JSONDecoder()
+    chunks: list[list] = []
+    pos = 0
+    length = len(text)
+
+    while pos < length:
+        # Skip whitespace and numeric length lines
+        while pos < length and text[pos] in " \t\r\n":
+            pos += 1
+        if pos >= length:
+            break
+
+        # Skip numeric length hints
+        if text[pos].isdigit():
+            while pos < length and text[pos] not in "\n":
+                pos += 1
+            continue
+
+        # Try to decode a JSON array
+        if text[pos] == "[":
+            try:
+                obj, end = decoder.raw_decode(text, pos)
+                chunks.append(obj)
+                pos = end
+            except json.JSONDecodeError:
+                pos += 1
+        else:
+            pos += 1
+
+    return chunks
+
+
+def decode_response(raw: bytes | str, rpc_id: str) -> Any:
+    """Decode a batchexecute response and extract the result for rpc_id.
+
+    Returns the parsed inner JSON, or None if the result is null.
+    Raises RPCError or AuthError on protocol-level errors.
+    """
+    text = _strip_prefix(raw)
+    chunks = _parse_chunks(text)
+
+    for chunk in chunks:
+        if not isinstance(chunk, list) or not chunk:
+            continue
+        for entry in chunk:
+            if not isinstance(entry, list) or len(entry) < 2:
+                continue
+
+            # Check for error entries
+            if entry[0] == "er" and len(entry) >= 2:
+                err_data = entry[1] if len(entry) > 1 else None
+                raise RPCError(f"RPC error: {err_data}", code=err_data)
+
+            # Look for the result matching our rpc_id
+            if entry[0] == "wrb.fr" and entry[1] == rpc_id:
+                raw_result = entry[2] if len(entry) > 2 else None
+                if raw_result is None:
+                    return None
+                # Double-parse: result is a JSON string inside JSON
+                return json.loads(raw_result)
+
+    return None

cli_web/codewiki/core/rpc/encoder.py

@@ -0,0 +1,32 @@
+"""Encode batchexecute RPC requests for Code Wiki."""
+
+from __future__ import annotations
+
+import json
+from urllib.parse import urlencode
+
+from .types import BATCHEXECUTE_URL
+
+
+def build_url(rpc_id: str) -> str:
+    """Build the full batchexecute URL with query parameters."""
+    params = {
+        "rpcids": rpc_id,
+        "source-path": "/",
+        "hl": "en",
+        "rt": "c",
+    }
+    return f"{BATCHEXECUTE_URL}?{urlencode(params)}"
+
+
+def encode_request(rpc_id: str, params: list) -> str:
+    """Encode an RPC call into a form-encoded f.req body.
+
+    The batchexecute wire format wraps each call as:
+      [[[rpc_id, json_string_params, null, "generic"]]]
+
+    No CSRF token needed — Code Wiki is fully public.
+    """
+    inner = [rpc_id, json.dumps(params), None, "generic"]
+    freq = json.dumps([[inner]])
+    return urlencode({"f.req": freq})

cli_web/codewiki/core/rpc/types.py

@@ -0,0 +1,27 @@
+"""RPC method IDs and constants for Code Wiki batchexecute API."""
+
+
+class RPCMethod:
+    """Batchexecute RPC method identifiers."""
+
+    FEATURED_REPOS = "nm8Fsb"
+    WIKI_PAGE = "VSX6ub"
+    SEARCH_REPOS = "vyWDAf"
+    CHAT = "EgIxfe"
+
+
+BATCHEXECUTE_URL = "https://codewiki.google/_/BoqAngularSdlcAgentsUi/data/batchexecute"
+
+BASE_URL = "https://codewiki.google"
+
+DEFAULT_HEADERS = {
+    "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
+    "x-same-domain": "1",
+    "Origin": BASE_URL,
+    "Referer": f"{BASE_URL}/",
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/146.0.0.0 Safari/537.36"
+    ),
+}