pyrepowiki-cli 0.1.0__py3-none-any.whl

repowiki/services/zread/client.py

@@ -0,0 +1,414 @@
+"""ZreadClient: RSC scraping + REST metadata + ask talk over httpx."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import random
+import ssl
+
+import httpx
+
+from repowiki.services.zread.flight import (
+    WikiInfo,
+    Page,
+    parse_wiki_data,
+    rewrite_callouts,
+    strip_frontmatter,
+)
+
+BASE = "https://zread.ai"
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+_TRANSIENT_STATUS = {408, 425, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529}
+
+
+class ZreadError(Exception):
+    """Base error for Zread."""
+
+
+class ZreadConnectionError(ZreadError):
+    """Failed to reach zread.ai."""
+
+
+class ZreadNotFoundError(ZreadError):
+    """Repo/page not indexed or not found on zread.ai."""
+
+
+class ZreadChallengeError(ZreadError):
+    """Cloudflare challenge or rate-limit (403/429/503)."""
+
+
+def _split(repo: str) -> tuple[str, str]:
+    owner, _, name = repo.partition("/")
+    if not name:
+        raise ValueError(f"invalid repo: {repo!r}")
+    return owner, name
+
+
+def _backoff(attempt: int) -> float:
+    return min(2 ** attempt + random.uniform(0, 1.5), 30.0)
+
+
+def _is_cert_error(exc: BaseException) -> bool:
+    while exc is not None:
+        if isinstance(exc, ssl.SSLCertVerificationError):
+            return True
+        exc = exc.__cause__ or exc.__context__
+    return False
+
+
+def _error_detail(resp: httpx.Response) -> str:
+    try:
+        data = resp.json()
+    except ValueError:
+        data = None
+    if isinstance(data, dict):
+        for key in ("msg", "detail", "message", "error"):
+            value = data.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+    return resp.text.strip()[:500]
+
+
+def _unwrap(resp: httpx.Response) -> object:
+    try:
+        result = resp.json()
+    except json.JSONDecodeError as exc:
+        raise ZreadError("invalid JSON response from Zread") from exc
+    if not isinstance(result, dict) or result.get("code") != 0:
+        msg = result.get("msg") if isinstance(result, dict) else "unknown"
+        raise ZreadError(f"Zread API error: {msg}")
+    return result.get("data")
+
+
+def _parse_sse_body(lines: list[str]) -> str:
+    event = None
+    finished: list[str] = []
+    chunks: list[str] = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        if line.startswith("event:"):
+            event = line[6:].strip()
+            if event == "finish":
+                break
+            if event == "error":
+                raise ZreadError("talk stream returned an error event")
+        elif line.startswith("data:"):
+            try:
+                data = json.loads(line[5:].strip())
+            except json.JSONDecodeError:
+                continue
+            if event == "round_finish":
+                finished.append(data.get("text", ""))
+            elif event == "answer":
+                chunks.append(data.get("text", ""))
+    text = "\n".join(finished) or "".join(chunks)
+    return text.strip()
+
+
+class ZreadClient:
+    """Stateless client; language/model/token are fixed per instance."""
+
+    def __init__(
+        self,
+        transport: httpx.AsyncBaseTransport | None = None,
+        lang: str = "en",
+        model: str = "glm-5.1",
+        token: str | None = None,
+        retries: int = 5,
+    ) -> None:
+        self._transport = transport
+        self._lang = lang
+        self._model = model
+        self._token = token
+        self._retries = retries
+
+    def _http(self, timeout: float) -> httpx.AsyncClient:
+        return httpx.AsyncClient(
+            timeout=timeout,
+            verify=True,
+            headers={"User-Agent": USER_AGENT},
+            cookies={"X-Locale": self._lang},
+            transport=self._transport,
+        )
+
+    def _headers(self, extra: dict | None = None) -> dict:
+        h = {"x-locale": self._lang}
+        if extra:
+            h.update(extra)
+        return h
+
+    async def _request(
+        self,
+        method: str,
+        url: str,
+        *,
+        timeout: float,
+        headers: dict | None = None,
+        params: dict | None = None,
+        json_body: dict | None = None,
+    ) -> httpx.Response:
+        async with self._http(timeout) as http:
+            for attempt in range(1, self._retries + 1):
+                try:
+                    resp = await http.request(
+                        method, url, headers=headers, params=params, json=json_body,
+                    )
+                except httpx.TransportError as exc:
+                    message = f"Failed to connect to Zread: {exc}"
+                    if _is_cert_error(exc):
+                        message += (
+                            " (TLS certificate verification failed; set SSL_CERT_FILE "
+                            "to your CA bundle to trust a proxy/mirror)"
+                        )
+                    if attempt >= self._retries or _is_cert_error(exc):
+                        raise ZreadConnectionError(message) from exc
+                    await asyncio.sleep(_backoff(attempt))
+                    continue
+                if resp.status_code == 429 and attempt < self._retries:
+                    wait = _backoff(attempt)
+                    try:
+                        wait = max(wait, float(resp.headers.get("Retry-After", "")))
+                    except ValueError:
+                        pass
+                    await asyncio.sleep(wait)
+                    continue
+                if resp.status_code in _TRANSIENT_STATUS and attempt < self._retries:
+                    await asyncio.sleep(_backoff(attempt))
+                    continue
+                if resp.status_code in (403, 429, 503):
+                    raise ZreadChallengeError(f"Zread returned HTTP {resp.status_code} (Cloudflare)")
+                if resp.status_code == 404:
+                    raise ZreadNotFoundError(f"Zread returned HTTP 404 for {url}")
+                if resp.status_code in _TRANSIENT_STATUS:
+                    raise ZreadConnectionError(
+                        f"Zread returned HTTP {resp.status_code} after {self._retries} attempts"
+                    )
+                if not resp.is_success:
+                    detail = _error_detail(resp)
+                    raise ZreadError(
+                        f"Zread returned HTTP {resp.status_code} for {url}"
+                        + (f": {detail}" if detail else "")
+                    )
+                return resp
+        raise ZreadError(f"Zread request failed after {self._retries} attempts")
+
+    async def repo_info(self, repo: str) -> dict:
+        owner, name = _split(repo)
+        resp = await self._request(
+            "GET", f"{BASE}/api/v1/repo/github/{owner}/{name}",
+            timeout=30.0, headers=self._headers(),
+        )
+        return _unwrap(resp) or {}
+
+    async def search_repos(self, query: str) -> list:
+        resp = await self._request(
+            "GET", f"{BASE}/api/v1/repo", timeout=30.0,
+            params={"q": query}, headers=self._headers(),
+        )
+        return _unwrap(resp) or []
+
+    async def trending(self) -> list:
+        resp = await self._request(
+            "GET", f"{BASE}/api/v1/public/repo/trending",
+            timeout=30.0, headers=self._headers(),
+        )
+        return _unwrap(resp) or []
+
+    async def recommend(self, topic: str = "") -> dict:
+        params = {"topic": topic} if topic else None
+        resp = await self._request(
+            "GET", f"{BASE}/api/v1/repo/recommend", timeout=30.0,
+            params=params, headers=self._headers(),
+        )
+        return _unwrap(resp) or {}
+
+    async def submit(self, repo: str) -> dict:
+        if not self._token:
+            raise ZreadError(
+                "ZREAD_TOKEN is required for submit. Get it from zread.ai after login: "
+                "run JSON.parse(localStorage.getItem('CGX_AUTH_STORAGE')).state.token in "
+                "the DevTools console, then export ZREAD_TOKEN."
+            )
+        resp = await self._request(
+            "POST", f"{BASE}/api/v1/public/repo/submit", timeout=30.0,
+            # The endpoint 400s without notification_email; upstream sends this placeholder.
+            json_body={"name_or_path": repo, "notification_email": "example@zread.ai"},
+            headers=self._headers({"Authorization": f"Bearer {self._token}"}),
+        )
+        return _unwrap(resp) or {}
+
+    async def search_wiki(self, repo: str, query: str) -> list:
+        info = await self.repo_info(repo)
+        wiki_id = str(info.get("wiki_id") or "")
+        if not wiki_id:
+            raise ZreadNotFoundError(f"no wiki_id for {repo}")
+        resp = await self._request(
+            "GET", f"{BASE}/api/v1/wiki/{wiki_id}/search", timeout=30.0,
+            params={"q": query}, headers=self._headers(),
+        )
+        return _unwrap(resp) or []
+
+    async def refresh(self, repo: str) -> dict:
+        info = await self.repo_info(repo)
+        repo_id = str(info.get("repo_id") or "")
+        if not repo_id:
+            raise ZreadNotFoundError(f"no repo_id for {repo}")
+        await self._request(
+            "POST", f"{BASE}/api/v1/repo/{repo_id}/refresh", timeout=30.0,
+            headers=self._headers(),
+        )
+        return {"repo_id": repo_id, "ok": True}
+
+    async def eta(self) -> dict:
+        resp = await self._request(
+            "GET", f"{BASE}/api/v1/repo/eta", timeout=30.0, headers=self._headers(),
+        )
+        return _unwrap(resp) or {}
+
+    async def github_head(self, repo: str) -> dict:
+        """Return the GitHub HEAD commit for a repo: ``{"sha": ..., "when": ...}``."""
+        owner, name = _split(repo)
+        resp = await self._request(
+            "GET", f"https://api.github.com/repos/{owner}/{name}/commits/HEAD",
+            timeout=30.0, headers=self._headers(),
+        )
+        try:
+            commit = resp.json()
+        except json.JSONDecodeError as exc:
+            raise ZreadError("invalid JSON response from GitHub") from exc
+        if not isinstance(commit, dict):
+            raise ZreadError("unexpected GitHub response")
+        committer = commit.get("commit", {}).get("committer") or {}
+        when = committer.get("date") if isinstance(committer, dict) else None
+        return {"sha": commit.get("sha") or "", "when": when}
+
+    async def read_file(
+        self, repo_id: str, path: str,
+        start: int | None = None, end: int | None = None,
+    ) -> str:
+        item = {"path": path}
+        if start is not None:
+            item["start_line"] = start
+        if end is not None:
+            item["end_line"] = end
+        resp = await self._request(
+            "POST", f"{BASE}/api/v1/repo/{repo_id}/files", timeout=30.0,
+            json_body={"files": [item]}, headers=self._headers(),
+        )
+        files = _unwrap(resp)
+        if not isinstance(files, list) or not files:
+            raise ZreadNotFoundError(f"file not found: {path}")
+        info = files[0]
+        snippet = info.get("snippet")
+        if isinstance(snippet, dict) and snippet.get("content") is not None:
+            return snippet["content"]
+        return info.get("content", "")
+
+    async def outline(self, repo: str) -> tuple[WikiInfo, list[Page]]:
+        wiki_id = await self._wiki_id(repo)
+        resp = await self._request(
+            "GET", f"{BASE}/api/v1/wiki/{wiki_id}", timeout=30.0, headers=self._headers(),
+        )
+        data = _unwrap(resp)
+        if not isinstance(data, dict):
+            raise ZreadNotFoundError(f"no wiki found for {repo}")
+        wiki_info, pages = parse_wiki_data(data)
+        if not pages:
+            raise ZreadNotFoundError(f"no wiki found for {repo}")
+        return wiki_info, pages
+
+    async def page(self, repo: str, slug: str | None = None) -> str:
+        wiki_id = await self._wiki_id(repo)
+        if slug is None:
+            _, pages = await self.outline(repo)
+            if not pages:
+                raise ZreadNotFoundError(f"no wiki pages for {repo}")
+            slug = pages[0].slug
+        resp = await self._request(
+            "GET", f"{BASE}/api/v1/wiki/{wiki_id}/page/{slug}", timeout=30.0,
+            headers=self._headers(),
+        )
+        data = _unwrap(resp)
+        if not isinstance(data, dict):
+            raise ZreadNotFoundError(f"no markdown found for {repo}/{slug}")
+        content = data.get("content")
+        if not isinstance(content, str) or not content.strip():
+            raise ZreadNotFoundError(f"no markdown found for {repo}/{slug}")
+        return rewrite_callouts(strip_frontmatter(content))
+
+    async def _wiki_id(self, repo: str) -> str:
+        info = await self.repo_info(repo)
+        wiki_id = str(info.get("wiki_id") or "")
+        if not wiki_id:
+            raise ZreadNotFoundError(f"no wiki_id for {repo}")
+        return wiki_id
+
+    async def ask(self, repo: str, question: str) -> str:
+        if not self._token:
+            raise ZreadError(
+                "ZREAD_TOKEN is required for ask. Get it from zread.ai after login: "
+                "run JSON.parse(localStorage.getItem('CGX_AUTH_STORAGE')).state.token in "
+                "the DevTools console, then export ZREAD_TOKEN."
+            )
+        info = await self.repo_info(repo)
+        repo_id = str(info.get("repo_id") or "")
+        wiki_id = str(info.get("wiki_id") or "")
+        if not repo_id or not wiki_id:
+            raise ZreadNotFoundError(f"{repo} is not indexed on zread.ai")
+        _, pages = await self.outline(repo)
+        if not pages:
+            raise ZreadNotFoundError(f"{repo} has no wiki pages")
+        talk_id = await self._create_talk()
+        return await self._send_message(talk_id, question, wiki_id, pages[0].page_id, repo_id)
+
+    async def _create_talk(self) -> str:
+        resp = await self._request(
+            "POST", f"{BASE}/api/v1/talk", timeout=30.0,
+            json_body={"query": " "},
+            headers=self._headers({"Authorization": f"Bearer {self._token}"}),
+        )
+        data = _unwrap(resp) or {}
+        talk_id = data.get("talk_id")
+        if not talk_id:
+            raise ZreadError("talk creation returned no talk_id")
+        return talk_id
+
+    async def _send_message(
+        self, talk_id: str, question: str,
+        wiki_id: str, page_id: str, repo_id: str,
+    ) -> str:
+        url = f"{BASE}/api/v1/talk/{talk_id}/message"
+        headers = self._headers({
+            "Authorization": f"Bearer {self._token}",
+            "Accept": "text/event-stream",
+        })
+        context = {"wiki": {"page_id": page_id, "wiki_id": wiki_id}}
+        if repo_id:
+            context["repo"] = {"repo_id": repo_id}
+        body = {
+            "parent_message_id": "",
+            "query": question,
+            "context": context,
+            "model": self._model,
+        }
+        async with self._http(timeout=120.0) as http:
+            async with http.stream("POST", url, headers=headers, json=body) as resp:
+                if resp.status_code in (403, 429, 503):
+                    raise ZreadChallengeError(f"Zread returned HTTP {resp.status_code}")
+                if resp.status_code == 404:
+                    raise ZreadNotFoundError(f"talk not found: {talk_id}")
+                if not resp.is_success:
+                    await resp.aread()
+                    detail = _error_detail(resp)
+                    raise ZreadError(
+                        f"Zread returned HTTP {resp.status_code} for {url}"
+                        + (f": {detail}" if detail else "")
+                    )
+                lines = [line async for line in resp.aiter_lines()]
+        return _parse_sse_body(lines)

repowiki/services/zread/flight.py

@@ -0,0 +1,116 @@
+"""Models and pure-function transforms for zread.ai wiki data."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass
+class Page:
+    page_id: str
+    slug: str
+    topic: str
+    section: str = ""
+    group: str = ""
+    order: int = 0
+
+
+@dataclass
+class WikiInfo:
+    wiki_id: str = ""
+    repo_id: str = ""
+
+
+_CALLOUT_RE = re.compile(r"<Cgx(?P<kind>[A-Za-z]+)>(?P<body>.*?)</Cgx(?P=kind)>", re.S)
+_FENCE_OPEN_RE = re.compile(r"^\s{0,3}(```+|~~~+)")
+
+_CALLOUT_KIND = {
+    "tip": "TIP",
+    "note": "NOTE",
+    "info": "NOTE",
+    "warn": "WARNING",
+    "warning": "WARNING",
+    "caution": "CAUTION",
+    "important": "CAUTION",
+    "danger": "CAUTION",
+}
+
+
+def parse_wiki_data(data: dict) -> tuple[WikiInfo, list[Page]]:
+    """Build ``(WikiInfo, pages)`` from the ``GET /api/v1/wiki/{wiki_id}`` body."""
+    info = data.get("info") if isinstance(data.get("info"), dict) else {}
+    wiki_info = WikiInfo(
+        wiki_id=str(info.get("wiki_id") or ""),
+        repo_id=str(info.get("repo_id") or ""),
+    )
+    pages = []
+    raw_pages = data.get("pages") if isinstance(data.get("pages"), list) else []
+    for i, p in enumerate(raw_pages):
+        if not isinstance(p, dict):
+            continue
+        slug = (p.get("slug") or "").strip()
+        if not slug:
+            continue
+        pages.append(
+            Page(
+                page_id=str(p.get("page_id") or ""),
+                slug=slug,
+                topic=(p.get("topic") or slug).strip(),
+                section=(p.get("section") or "").strip(),
+                group=(p.get("group") or "").strip(),
+                order=int(p.get("order", i) or 0),
+            )
+        )
+    pages.sort(key=lambda p: p.order)
+    for i, p in enumerate(pages):
+        p.order = i
+    return wiki_info, pages
+
+
+def _code_spans(md: str) -> list[tuple[int, int]]:
+    spans = []
+    pos = 0
+    fence = None
+    start = 0
+    for line in md.splitlines(keepends=True):
+        stripped = line.rstrip("\n")
+        if fence is None:
+            m = _FENCE_OPEN_RE.match(stripped)
+            if m:
+                fence = m.group(1)[0] * 3
+                start = pos
+        elif stripped.strip().startswith(fence):
+            spans.append((start, pos + len(line)))
+            fence = None
+        pos += len(line)
+    if fence is not None:
+        spans.append((start, len(md)))
+    return spans
+
+
+def _in_spans(idx: int, spans: list[tuple[int, int]]) -> bool:
+    return any(a <= idx < b for a, b in spans)
+
+
+def rewrite_callouts(md: str) -> str:
+    """Convert ``<CgxTip>…</CgxTip>`` shells to GitHub ``> [!TIP]`` blocks."""
+    spans = _code_spans(md)
+
+    def sub(m: re.Match) -> str:
+        if _in_spans(m.start(), spans):
+            return m.group(0)
+        kind = _CALLOUT_KIND.get(m.group("kind").lower(), "NOTE")
+        body = m.group("body").strip()
+        if not body:
+            return ""
+        lines = "\n".join(f"> {ln}".rstrip() for ln in body.split("\n"))
+        return f"> [!{kind}]\n{lines}"
+
+    return _CALLOUT_RE.sub(sub, md)
+
+
+def strip_frontmatter(md: str) -> str:
+    """Remove a leading ``---\n…\n---`` YAML frontmatter block."""
+    m = re.match(r"^---\n.*?\n---\n\n?", md, re.S)
+    return md[m.end():] if m else md

repowiki/shared/__init__.py

@@ -0,0 +1 @@
+"""Shared plumbing for all repowiki services."""

repowiki/shared/async_.py

@@ -0,0 +1,15 @@
+"""Synchronous bridge for async client calls.
+
+Named ``async_`` (not ``async``) because ``async`` is a Python keyword and a
+module by that name cannot be reached with a normal ``import`` statement.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+
+def run_async(coro: Any) -> Any:
+    """Run an async coroutine synchronously."""
+    return asyncio.run(coro)

repowiki/shared/export.py

@@ -0,0 +1,36 @@
+"""Export wiki pages as Markdown files plus llms.txt / README.md indexes."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+
+def _safe_name(title: str, index: int) -> str:
+    """Turn a page title into a filesystem-safe ``NN-name.md`` filename."""
+    slug = re.sub(r"[^\w\-]+", "-", title.strip()).strip("-").lower() or "page"
+    return f"{index:02d}-{slug}.md"
+
+
+def export_pages(out_dir: str, repo: str, pages: list[tuple[str, str]], full: str) -> int:
+    """Write one file per page plus ``llms.txt`` / ``README.md`` / ``llms-full.txt``.
+
+    ``pages`` is a list of ``(title, markdown)``; ``full`` is the complete
+    concatenated markdown. ``llms.txt`` and ``README.md`` carry the same page
+    index (``README.md`` for GitHub auto-rendering). Returns the page count.
+    """
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    entries: list[tuple[str, str]] = []
+    for i, (title, markdown) in enumerate(pages):
+        fname = _safe_name(title, i)
+        (out / fname).write_text(markdown, encoding="utf-8")
+        entries.append((title, fname))
+    index = ["# " + repo, ""]
+    for title, fname in entries:
+        index.append(f"- [{title}]({fname})")
+    index_text = "\n".join(index) + "\n"
+    (out / "llms.txt").write_text(index_text, encoding="utf-8")
+    (out / "README.md").write_text(index_text, encoding="utf-8")
+    (out / "llms-full.txt").write_text(full + "\n", encoding="utf-8")
+    return len(entries)

repowiki/shared/model.py

@@ -0,0 +1,29 @@
+"""Shared result types for wiki backends."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Reference:
+    file_path: str
+    range_start: int | None = None
+    range_end: int | None = None
+
+
+@dataclass
+class SourceFile:
+    repo: str
+    path: str
+    content: str
+
+
+@dataclass
+class Answer:
+    body: str
+    summary: str | None = None
+    references: list[Reference] = field(default_factory=list)
+    sources: list[SourceFile] = field(default_factory=list)
+    stats: dict[str, float] = field(default_factory=dict)
+    query_id: str | None = None