pyrepowiki-cli 0.1.0__py3-none-any.whl

repowiki/services/deepwiki/client.py

@@ -0,0 +1,164 @@
+"""MCP client wrapper for the DeepWiki API."""
+
+from __future__ import annotations
+
+import logging
+import os
+from contextlib import AsyncExitStack
+from typing import Any
+
+import httpx
+from mcp import ClientSession
+from mcp.client.streamable_http import streamablehttp_client
+
+from repowiki.shared.model import Answer
+
+logging.getLogger("httpx_sse").setLevel(logging.ERROR)
+logging.getLogger("mcp.client.streamable_http").setLevel(logging.ERROR)
+
+DEFAULT_MCP_URL = "https://mcp.deepwiki.com/mcp"
+
+
+class DeepWikiError(Exception):
+    """Base exception for DeepWiki errors."""
+
+
+class ConnectionError(DeepWikiError):
+    """Failed to connect to the DeepWiki server."""
+
+
+class ToolError(DeepWikiError):
+    """Error returned from an MCP tool execution."""
+
+
+def _root_cause(exc: BaseException) -> BaseException:
+    """Unwrap an exception group (Python 3.11+) to its first leaf cause."""
+    while (sub := getattr(exc, "exceptions", None)):
+        exc = sub[0]
+    return exc
+
+
+def _is_connection_failure(exc: BaseException) -> bool:
+    if isinstance(exc, (OSError, httpx.TransportError)):
+        return True
+    message = str(exc).lower()
+    return "connection" in message or "timeout" in message
+
+
+class DeepWikiClient:
+    """Client for interacting with DeepWiki via MCP.
+
+    Can be used one-shot (each call opens and closes its own connection) or as
+    a persistent session via the async context manager, which reuses a single
+    connection across calls — the REPL uses this to avoid re-handshaking per
+    question.
+    """
+
+    def __init__(self, base_url: str | None = None) -> None:
+        self.base_url = base_url or os.environ.get("DEEPWIKI_MCP_URL", DEFAULT_MCP_URL)
+        self._stack: AsyncExitStack | None = None
+        self._session: ClientSession | None = None
+
+    # -- persistent session management -------------------------------------
+
+    async def __aenter__(self) -> "DeepWikiClient":
+        await self.open()
+        return self
+
+    async def __aexit__(self, *exc_info: object) -> None:
+        await self.close()
+
+    async def open(self) -> None:
+        """Open and initialize a session, retrying once on connection failure."""
+        last: ConnectionError | None = None
+        for _ in range(2):
+            try:
+                await self._open_once()
+                return
+            except ConnectionError as exc:
+                last = exc
+        assert last is not None
+        raise last
+
+    async def close(self) -> None:
+        """Close the open session, if any."""
+        stack, self._stack = self._stack, None
+        self._session = None
+        if stack is not None:
+            await stack.aclose()
+
+    async def _open_once(self) -> None:
+        stack = AsyncExitStack()
+        try:
+            read, write, _ = await stack.enter_async_context(
+                streamablehttp_client(self.base_url)
+            )
+            session = await stack.enter_async_context(ClientSession(read, write))
+            await session.initialize()
+        except Exception as exc:
+            await stack.aclose()
+            root = _root_cause(exc)
+            if _is_connection_failure(root):
+                raise ConnectionError(
+                    f"Failed to connect to DeepWiki server: {root}"
+                ) from exc
+            raise ToolError(f"Failed to initialize MCP session: {root}") from exc
+        self._stack = stack
+        self._session = session
+
+    # -- tool calls ---------------------------------------------------------
+
+    async def _invoke(self, tool_name: str, arguments: dict[str, Any]) -> str:
+        assert self._session is not None
+        try:
+            result = await self._session.call_tool(tool_name, arguments)
+        except Exception as exc:
+            root = _root_cause(exc)
+            if _is_connection_failure(root):
+                raise ConnectionError(
+                    f"Failed to connect to DeepWiki server: {root}"
+                ) from exc
+            raise ToolError(f"Tool '{tool_name}' failed: {root}") from exc
+        return self._extract_text_content(result)
+
+    async def _call_oneshot(self, tool_name: str, arguments: dict[str, Any]) -> str:
+        try:
+            await self.open()
+            return await self._invoke(tool_name, arguments)
+        finally:
+            await self.close()
+
+    async def _call_persistent(self, tool_name: str, arguments: dict[str, Any]) -> str:
+        for attempt in range(2):
+            try:
+                return await self._invoke(tool_name, arguments)
+            except ConnectionError as exc:
+                if attempt == 0:
+                    await self.close()
+                    await self.open()
+                    continue
+                raise
+
+    def _extract_text_content(self, result: Any) -> str:
+        parts = [c.text for c in result.content if getattr(c, "type", None) == "text"]
+        text = "\n".join(parts)
+        if result.isError:
+            raise ToolError(text or "Unknown tool error")
+        return text
+
+    async def _call_tool(self, tool_name: str, arguments: dict[str, Any]) -> str:
+        if self._session is None:
+            return await self._call_oneshot(tool_name, arguments)
+        return await self._call_persistent(tool_name, arguments)
+
+    async def read_wiki_structure(self, repo_name: str) -> str:
+        return await self._call_tool("read_wiki_structure", {"repoName": repo_name})
+
+    async def read_wiki_contents(self, repo_name: str) -> str:
+        return await self._call_tool("read_wiki_contents", {"repoName": repo_name})
+
+    async def ask_question(self, repo_name: str, question: str) -> Answer:
+        text = await self._call_tool(
+            "ask_question", {"repoName": repo_name, "question": question}
+        )
+        return Answer(body=text)

repowiki/services/deepwiki/codemap.py

@@ -0,0 +1,74 @@
+"""Codemap JSON → Mermaid flowchart rendering."""
+
+from __future__ import annotations
+
+import json
+import re
+
+_TRACE_COLORS = [
+    ("#e8f5e9", "#4caf50"),
+    ("#e3f2fd", "#2196f3"),
+    ("#fff3e0", "#ff9800"),
+    ("#f3e5f5", "#9c27b0"),
+    ("#fff8e1", "#ffc107"),
+    ("#fce4ec", "#e91e63"),
+    ("#e0f2f1", "#009688"),
+    ("#fbe9e7", "#ff5722"),
+]
+
+
+def _sanitize_id(value: object) -> str:
+    return re.sub(r"[^a-zA-Z0-9]", "_", str(value))
+
+
+def _escape_label(value: str) -> str:
+    return value.replace('"', "#quot;").replace("\n", " ")
+
+
+def _short_path(path: str) -> str:
+    return path.rsplit("/", 1)[-1]
+
+
+def codemap_to_mermaid(text: str) -> str | None:
+    """Return a Mermaid flowchart for a codemap JSON blob, else None."""
+    try:
+        data = json.loads(text)
+    except (json.JSONDecodeError, TypeError):
+        return None
+    if not isinstance(data, dict) or not isinstance(data.get("traces"), list):
+        return None
+    return _render(data["traces"])
+
+
+def _render(traces: list[dict]) -> str:
+    lines = ["flowchart TB"]
+    for i, trace in enumerate(traces):
+        sg_id = _sanitize_id(f"trace_{trace.get('id', i)}")
+        title = _escape_label(str(trace.get("title", "")))
+        lines.append("")
+        lines.append(f'    subgraph {sg_id}["{trace.get("id", i)}. {title}"]')
+        locations = trace.get("locations", [])
+        for loc in locations:
+            loc_id = _sanitize_id(f"loc_{loc.get('id')}")
+            loc_title = _escape_label(str(loc.get("title", "")))
+            filename = _short_path(str(loc.get("path", "")))
+            label = f"{loc_title}\\n{filename}:{loc.get('lineNumber', '')}"
+            lines.append(f'        {loc_id}["{label}"]')
+        lines.append("    end")
+        for j in range(len(locations) - 1):
+            a = _sanitize_id(f"loc_{locations[j].get('id')}")
+            b = _sanitize_id(f"loc_{locations[j + 1].get('id')}")
+            lines.append(f"    {a} --> {b}")
+    for i in range(len(traces) - 1):
+        curr = traces[i].get("locations", [])
+        nxt = traces[i + 1].get("locations", [])
+        if curr and nxt:
+            a = _sanitize_id(f"loc_{curr[-1].get('id')}")
+            b = _sanitize_id(f"loc_{nxt[0].get('id')}")
+            lines.append(f"    {a} -.-> {b}")
+    lines.append("")
+    for i, trace in enumerate(traces):
+        sg_id = _sanitize_id(f"trace_{trace.get('id', i)}")
+        fill, stroke = _TRACE_COLORS[i % len(_TRACE_COLORS)]
+        lines.append(f"    style {sg_id} fill:{fill},stroke:{stroke},stroke-width:2px")
+    return "\n".join(lines)

repowiki/services/deepwiki/devin.py

@@ -0,0 +1,355 @@
+"""Reverse-engineered api.devin.ai client."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import re
+import ssl
+import time
+from collections.abc import Callable
+from contextlib import nullcontext
+from uuid import uuid4
+
+import httpx
+import websockets
+from websockets.exceptions import ConnectionClosedOK, WebSocketException
+
+from repowiki.services.deepwiki.client import ConnectionError, ToolError
+from repowiki.shared.model import Answer, Reference, SourceFile
+
+
+_CITE_RE = re.compile(r"<cite\s+([^>]*?)\s*/?>")
+_CITE_ATTR_RE = re.compile(r'(\w+)="([^"]*)"')
+
+
+def _parse_cites(body: str) -> tuple[str, list[Reference]]:
+    """Replace inline ``<cite …/>`` tags with ``[i]`` markers, returning references.
+
+    Some responses encode citations as self-closing tags carrying ``path`` and
+    ``start`` (an ``X-Y`` line range) instead of separate ``reference`` events.
+    """
+    references: list[Reference] = []
+
+    def _replace(match: re.Match[str]) -> str:
+        attrs = dict(_CITE_ATTR_RE.findall(match.group(1)))
+        path = attrs.get("path", "")
+        if not path:
+            return match.group(0)
+        parts = attrs.get("start", "").split("-")
+        try:
+            range_start = int(parts[0])
+            range_end = int(parts[1]) if len(parts) > 1 else range_start
+        except (ValueError, IndexError):
+            range_start = range_end = None
+        references.append(Reference(path, range_start, range_end))
+        return f"[{len(references)}]"
+
+    return _CITE_RE.sub(_replace, body), references
+
+
+def parse_response(query: dict, query_id: str | None) -> Answer:
+    """Assemble an Answer from a single query block's response events."""
+    body: list[str] = []
+    summary: list[str] = []
+    references: list[Reference] = []
+    sources: list[SourceFile] = []
+    seen: set[tuple[str, str]] = set()
+    stats: dict[str, float] = {}
+
+    for event in query.get("response", []):
+        kind = event.get("type")
+        data = event.get("data")
+        if kind == "chunk":
+            body.append(data)
+        elif kind == "summary_chunk":
+            summary.append(data)
+        elif kind == "reference":
+            references.append(
+                Reference(
+                    file_path=data["file_path"],
+                    range_start=data.get("range_start"),
+                    range_end=data.get("range_end"),
+                )
+            )
+            body.append(f"[{len(references)}]")
+        elif kind == "file_contents":
+            repo, path, content = data
+            key = (repo, path)
+            if key not in seen:
+                seen.add(key)
+                sources.append(SourceFile(repo=repo, path=path, content=content))
+        elif kind == "stats":
+            stats[data["key"]] = data["value"]
+        elif kind == "done":
+            break
+
+    body_text = "".join(body)
+    if not references and "<cite" in body_text:
+        body_text, references = _parse_cites(body_text)
+
+    return Answer(
+        body=body_text,
+        summary="".join(summary) or None,
+        references=references,
+        sources=sources,
+        stats=stats,
+        query_id=query_id,
+    )
+
+
+DEFAULT_API_URL = "https://api.devin.ai"
+
+ENGINE_MAP = {"fast": "multihop_faster", "deep": "agent", "codemap": "codemap"}
+
+
+def _http_detail(exc: httpx.HTTPStatusError) -> str:
+    """Surface FastAPI's ``{"detail": ...}`` body so the CLI can classify it."""
+    try:
+        detail = exc.response.json().get("detail")
+    except Exception:
+        return ""
+    if isinstance(detail, str) and detail:
+        return f": {detail}"
+    return ""
+
+
+def _is_cert_error(exc: BaseException) -> bool:
+    """True if any cause in the exception chain is a TLS cert verification error."""
+    while exc is not None:
+        if isinstance(exc, ssl.SSLCertVerificationError):
+            return True
+        exc = exc.__cause__ or exc.__context__
+    return False
+
+
+def _connection_message(host: str, exc: httpx.TransportError) -> str:
+    """Build a connection error message, naming the host and hinting on TLS failure."""
+    message = f"Failed to connect to {host}: {exc}"
+    if _is_cert_error(exc):
+        message += (
+            " (TLS certificate verification failed; set SSL_CERT_FILE "
+            "to your CA bundle to trust a proxy/mirror)"
+        )
+    return message
+
+
+class DevinClient:
+    """Client for the reverse-engineered api.devin.ai Q&A endpoints."""
+
+    def __init__(self, base_url: str | None = None) -> None:
+        self.base_url = base_url or os.environ.get("DEEPWIKI_API_URL", DEFAULT_API_URL)
+
+    async def _get_json(
+        self,
+        path: str,
+        *,
+        params: dict | None = None,
+        client: httpx.AsyncClient | None = None,
+    ) -> dict:
+        ctx = nullcontext(client) if client is not None else httpx.AsyncClient(
+            base_url=self.base_url, timeout=30.0
+        )
+        try:
+            async with ctx as c:
+                resp = await c.get(path, params=params)
+                resp.raise_for_status()
+                return resp.json()
+        except httpx.TransportError as exc:
+            raise ConnectionError(_connection_message("Devin server", exc)) from exc
+        except httpx.HTTPStatusError as exc:
+            raise ToolError(
+                f"Devin API returned HTTP {exc.response.status_code}{_http_detail(exc)}"
+            ) from exc
+
+    async def _post_json(
+        self,
+        path: str,
+        *,
+        params: dict | None = None,
+        json: dict | None = None,
+        client: httpx.AsyncClient | None = None,
+    ) -> dict:
+        ctx = nullcontext(client) if client is not None else httpx.AsyncClient(
+            base_url=self.base_url, timeout=30.0
+        )
+        try:
+            async with ctx as c:
+                resp = await c.post(path, params=params, json=json)
+                resp.raise_for_status()
+                return resp.json()
+        except httpx.TransportError as exc:
+            raise ConnectionError(_connection_message("Devin server", exc)) from exc
+        except httpx.HTTPStatusError as exc:
+            raise ToolError(
+                f"Devin API returned HTTP {exc.response.status_code}{_http_detail(exc)}"
+            ) from exc
+
+    async def ask(
+        self,
+        repos: list[str],
+        question: str,
+        *,
+        mode: str = "fast",
+        query_id: str | None = None,
+        timeout: float = 120.0,
+        poll_interval: float = 2.0,
+        context: str = "",
+        generate_summary: bool = True,
+        on_chunk: Callable[[str], None] | None = None,
+    ) -> Answer:
+        engine_id = ENGINE_MAP.get(mode)
+        if engine_id is None:
+            raise ToolError(f"unknown mode {mode!r}")
+        qid = query_id or str(uuid4())
+        payload = {
+            "engine_id": engine_id,
+            "user_query": question,
+            "keywords": [],
+            "repo_names": list(repos),
+            "additional_context": context,
+            "query_id": qid,
+            "use_notes": False,
+            "attached_context": [],
+            "generate_summary": generate_summary,
+        }
+        async with httpx.AsyncClient(base_url=self.base_url, timeout=30.0) as client:
+            await self._post_json("/ada/query", json=payload, client=client)
+            if on_chunk is not None:
+                events = await self._stream_chunks(qid, on_chunk, timeout=timeout)
+                return parse_response({"response": events}, qid)
+            return await self._poll_query(
+                qid, client=client, poll_interval=poll_interval, timeout=timeout
+            )
+
+    async def _stream_chunks(
+        self, qid: str, on_chunk: Callable[[str], None], *, timeout: float
+    ) -> list[dict]:
+        """Stream events over the WebSocket, emitting answer text, until 'done'.
+
+        Chunk text and inline citation markers (``[i]`` emitted at each
+        ``reference`` event) are forwarded through ``on_chunk``, so the streamed
+        body shows citations inline and lines up with the final ``## Sources``.
+
+        Returns the full event list so the caller can assemble a complete
+        Answer (body, summary, references, sources) without a follow-up GET.
+        """
+        ws_url = f"{self.base_url.replace('http', 'ws', 1)}/ada/ws/query/{qid}"
+        deadline = time.monotonic() + timeout
+        events: list[dict] = []
+        ref_count = 0
+        try:
+            async with websockets.connect(ws_url) as ws:
+                while True:
+                    remaining = deadline - time.monotonic()
+                    if remaining <= 0:
+                        raise ToolError("timed out waiting for answer")
+                    try:
+                        raw = await asyncio.wait_for(ws.recv(), timeout=remaining)
+                    except asyncio.TimeoutError:
+                        raise ToolError("timed out waiting for answer")
+                    except ConnectionClosedOK:
+                        break
+                    try:
+                        msg = json.loads(raw)
+                    except json.JSONDecodeError:
+                        continue
+                    if not isinstance(msg, dict):
+                        continue
+                    kind = msg.get("type")
+                    if kind is None:
+                        continue
+                    if kind == "done":
+                        break
+                    events.append(msg)
+                    if kind == "chunk":
+                        on_chunk(msg.get("data", ""))
+                    elif kind == "reference":
+                        ref_count += 1
+                        on_chunk(f"[{ref_count}]")
+        except (OSError, WebSocketException) as exc:
+            raise ConnectionError(f"WebSocket connection failed: {exc}") from exc
+        return events
+
+    async def _poll_query(
+        self,
+        qid: str,
+        *,
+        client: httpx.AsyncClient,
+        poll_interval: float,
+        timeout: float,
+    ) -> Answer:
+        deadline = time.monotonic() + timeout
+        while True:
+            await asyncio.sleep(poll_interval)
+            if time.monotonic() > deadline:
+                raise ToolError("timed out waiting for answer")
+            data = await self._get_json(f"/ada/query/{qid}", client=client)
+            queries = data.get("queries")
+            if not queries:
+                raise ToolError("Devin API returned no query results")
+            query = queries[-1]
+            if query.get("state") in ("done", "error"):
+                break
+        if query.get("error"):
+            raise ToolError(str(query["error"]))
+        return parse_response(query, qid)
+
+    async def list_public_indexes(self, search: str) -> dict:
+        return await self._get_json("/ada/list_public_indexes", params={"search_repo": search})
+
+    async def public_repo_indexing_status(self, repo: str) -> dict:
+        return await self._get_json("/ada/public_repo_indexing_status", params={"repo_name": repo})
+
+    async def warm_public_repo(self, repo: str) -> dict:
+        return await self._post_json("/ada/warm_public_repo", params={"repo_name": repo})
+
+    async def repo_index(self, repo: str) -> dict | None:
+        """Return the ``list_public_indexes`` entry matching ``repo`` exactly.
+
+        The search is fuzzy, so filter to the entry whose ``repo_name`` equals
+        the requested repo; return ``None`` when the repo has no index.
+        """
+        data = await self.list_public_indexes(repo)
+        for entry in data.get("indices") or []:
+            if entry.get("repo_name") == repo:
+                return entry
+        return None
+
+    async def github_head(self, repo: str) -> dict:
+        """Return the GitHub HEAD sha and commit date for ``repo``.
+
+        ``id`` in the index (e.g. ``…/IDPFold2/5315b279``) ends in a short
+        commit sha, so compare it against GitHub HEAD to detect a stale wiki.
+        """
+        owner, name = repo.split("/", 1)
+        try:
+            async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+                resp = await client.get(
+                    f"https://api.github.com/repos/{owner}/{name}/commits/HEAD"
+                )
+                resp.raise_for_status()
+                try:
+                    commit = resp.json()
+                except json.JSONDecodeError as exc:
+                    raise ToolError("invalid JSON response from GitHub") from exc
+        except httpx.TransportError as exc:
+            raise ConnectionError(_connection_message("GitHub", exc)) from exc
+        except httpx.HTTPStatusError as exc:
+            raise ToolError(f"GitHub API returned HTTP {exc.response.status_code}") from exc
+        if not isinstance(commit, dict):
+            raise ToolError("unexpected GitHub response")
+        committer = commit.get("commit", {}).get("committer") or {}
+        when = committer.get("date") if isinstance(committer, dict) else None
+        return {"sha": commit.get("sha") or "", "when": when}
+
+    async def get_query(self, query_id: str) -> Answer:
+        data = await self._get_json(f"/ada/query/{query_id}")
+        queries = data.get("queries")
+        if not queries:
+            raise ToolError("Devin API returned no query results")
+        query = queries[-1]
+        if query.get("error"):
+            raise ToolError(str(query["error"]))
+        return parse_response(query, query_id)

repowiki/services/zread/__init__.py

@@ -0,0 +1,5 @@
+"""Zread service: query zread.ai docs for public GitHub repositories."""
+
+from repowiki.services.zread.cli import register
+
+__all__ = ["register"]