thinkstack-core 4.0.0__py3-none-any.whl

thinkstack_core/cloud/mcp_client.py

@@ -0,0 +1,229 @@
+"""
+thinkstack_core.cloud.mcp_client
+==============================
+Reusable client for calling remote ThinkStack MCP tools via JSON-RPC over SSE.
+
+The cloud MCP server accepts tool requests via POST to the messages endpoint
+and returns the JSON-RPC results as SSE ``message`` events. This client keeps
+the SSE connection open in a background reader thread, posts requests, and
+matches responses by request ID.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+import time
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("thinkstack.cloud.mcp_client")
+
+_USER_AGENT = "thinkstack-mcp-client"
+
+
+class McpClient:
+    """Call remote ThinkStack MCP tools via JSON-RPC over SSE.
+
+    Usage::
+
+        client = McpClient(url, api_key)
+        result = client.call_tool("thinkstack_topics_list", {"all": True})
+        client.close()
+    """
+
+    def __init__(self, url: str, api_key: str, timeout: float = 10.0) -> None:
+        self._sse_url = url
+        self._api_key = api_key
+        self._timeout = timeout
+        self._messages_url: str | None = None
+        self._resp: Any | None = None
+        self._reader_thread: threading.Thread | None = None
+        self._results: dict[int | str, dict] = {}
+        self._lock = threading.Lock()
+        self._cv = threading.Condition(self._lock)
+        self._next_id = 1
+        self._closed = False
+
+    def __enter__(self) -> "McpClient":
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.close()
+
+    def call_tool(self, tool_name: str, args: dict | None = None) -> str:
+        """Call a remote MCP tool and return the result text.
+
+        Raises ``RuntimeError`` on network errors, timeouts, or MCP-level errors.
+        """
+        self._ensure_connected()
+        req_id = self._next_id
+        self._next_id += 1
+
+        payload = {
+            "jsonrpc": "2.0",
+            "id": req_id,
+            "method": "tools/call",
+            "params": {"name": tool_name, "arguments": args or {}},
+        }
+
+        self._post(payload)
+        body = self._wait_for_response(req_id)
+
+        if body.get("error"):
+            raise RuntimeError(f"MCP error: {json.dumps(body['error'])[:300]}")
+        if body.get("result", {}).get("isError"):
+            raise RuntimeError(f"MCP tool error: {json.dumps(body['result'])[:300]}")
+
+        content = body.get("result", {}).get("content", [])
+        if content and isinstance(content, list):
+            return content[0].get("text", "")
+        return json.dumps(body.get("result", {}))
+
+    def is_available(self) -> bool:
+        """Quick connectivity check — returns True if the SSE handshake succeeds."""
+        try:
+            self._ensure_connected()
+            return self._messages_url is not None
+        except Exception:
+            return False
+
+    def close(self) -> None:
+        """Close the SSE connection and stop the reader thread."""
+        self._closed = True
+        resp = self._resp
+        self._resp = None
+        if resp is not None:
+            try:
+                resp.close()
+            except Exception:
+                pass
+        with self._cv:
+            self._cv.notify_all()
+        reader = self._reader_thread
+        self._reader_thread = None
+        if reader is not None and reader.is_alive():
+            reader.join(timeout=2.0)
+
+    def _ensure_connected(self) -> None:
+        """Open the SSE connection and start the reader thread if needed."""
+        if self._closed:
+            raise RuntimeError("McpClient is closed")
+        if self._resp is not None and self._messages_url is not None:
+            return
+
+        req = urllib.request.Request(
+            self._sse_url,
+            headers={
+                "Accept": "text/event-stream",
+                "X-ThinkStack-Key": self._api_key,
+                "User-Agent": _USER_AGENT,
+            },
+        )
+        self._resp = urllib.request.urlopen(req, timeout=self._timeout)
+        self._reader_thread = threading.Thread(target=self._read_loop, daemon=True)
+        self._reader_thread.start()
+
+        # Wait for the endpoint event to set the messages URL.
+        deadline = time.time() + self._timeout
+        with self._cv:
+            while self._messages_url is None:
+                remaining = deadline - time.time()
+                if remaining <= 0:
+                    raise RuntimeError("SSE endpoint event not received in time")
+                self._cv.wait(remaining)
+
+    def _read_loop(self) -> None:
+        """Background thread: parse SSE events and store message responses."""
+        event_type: str | None = None
+        data: str | None = None
+        try:
+            while self._resp is not None:
+                line = self._resp.readline()
+                if not line:
+                    break
+                line = line.decode("utf-8", errors="replace")
+                if line.startswith("event:"):
+                    event_type = line[6:].strip()
+                elif line.startswith("data:"):
+                    data = line[5:].strip()
+                elif line.strip() == "":
+                    self._handle_event(event_type, data)
+                    event_type = None
+                    data = None
+        except Exception as exc:
+            logger.debug("SSE read loop ended: %s", exc)
+        finally:
+            self._resp = None
+            self._messages_url = None
+
+    def _handle_event(self, event_type: str | None, data: str | None) -> None:
+        """Process a single SSE event."""
+        if event_type == "endpoint" and data:
+            with self._cv:
+                self._messages_url = data
+                self._cv.notify_all()
+        elif event_type == "message" and data:
+            try:
+                body = json.loads(data)
+            except json.JSONDecodeError:
+                return
+            req_id = body.get("id")
+            if req_id is None:
+                return
+            with self._cv:
+                self._results[req_id] = body
+                self._cv.notify_all()
+
+    def _post(self, payload: dict) -> None:
+        """POST a JSON-RPC request to the messages endpoint."""
+        if self._messages_url is None:
+            raise RuntimeError("Message endpoint not available")
+        req = urllib.request.Request(
+            self._messages_url,
+            data=json.dumps(payload).encode(),
+            headers={
+                "Content-Type": "application/json",
+                "X-ThinkStack-Key": self._api_key,
+                "User-Agent": _USER_AGENT,
+            },
+        )
+        with urllib.request.urlopen(req, timeout=self._timeout) as resp:
+            # The server returns 202 Accepted; the actual response arrives via SSE.
+            pass
+
+    def _wait_for_response(self, req_id: int) -> dict:
+        """Wait for the JSON-RPC response matching ``req_id``."""
+        deadline = time.time() + self._timeout
+        with self._cv:
+            while req_id not in self._results:
+                remaining = deadline - time.time()
+                if remaining <= 0:
+                    raise RuntimeError(f"MCP call timed out waiting for response {req_id}")
+                self._cv.wait(remaining)
+            return self._results.pop(req_id)
+
+
+def detect_mcp_config(project_root: Path) -> tuple[str, str] | None:
+    """Check env vars then .mcp.json for MCP server URL and API key.
+
+    Returns ``(url, api_key)`` or ``None`` if not configured.
+    """
+    url = os.environ.get("THINKSTACK_MCP_URL", "").strip()
+    api_key = os.environ.get("THINKSTACK_API_KEY", "").strip()
+
+    if not url or not api_key:
+        mcp_path = project_root / ".mcp.json"
+        try:
+            data = json.loads(mcp_path.read_text(encoding="utf-8"))
+            env_cfg = data.get("mcpServers", {}).get("thinkstack", {}).get("env", {})
+            url = url or env_cfg.get("THINKSTACK_MCP_URL", "").strip()
+            api_key = api_key or env_cfg.get("THINKSTACK_API_KEY", "").strip()
+        except (OSError, json.JSONDecodeError, KeyError, TypeError):
+            pass
+
+    if url and api_key:
+        return (url, api_key)
+    return None

thinkstack_core/cloud/setup.py

@@ -0,0 +1,144 @@
+"""One-command Cloudflare Worker deployment for ThinkStack MCP server.
+
+This module wraps ``deploy/cloudflare-worker/setup-mcp-server.sh`` so it can be
+called as ``thinkstack cloud setup``. It is intentionally thin: the shell script
+still owns the npm/wrangler lifecycle; Python only validates CLI arguments and
+forwards them.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Sequence
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_PACKAGE_DIR = Path(__file__).resolve().parent
+_REPO_ROOT = _PACKAGE_DIR.parents[1]
+SETUP_SCRIPT = _REPO_ROOT / "deploy" / "cloudflare-worker" / "setup-mcp-server.sh"
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def run_cloud_setup(
+    *,
+    env_file: str | Path | None = None,
+    org: str | None = None,
+    repo: str | None = None,
+    ide: str | None = None,
+    openai_key: str | None = None,
+    non_interactive: bool = False,
+    skip_key: bool = False,
+    skip_client_config: bool = False,
+    dry_run: bool = False,
+) -> dict[str, str | list[str]]:
+    """Build and optionally run the Cloudflare Worker setup command.
+
+    Args:
+        env_file: Path to an environment file to source before running the script.
+        org: ThinkStack org ID. Required in non-interactive mode.
+        repo: ThinkStack repo ID. Required in non-interactive mode.
+        ide: IDE to configure (opencode, claude, cursor, vscode, stdio).
+        openai_key: Optional OpenAI API key for the DRPL embedding backend.
+        non_interactive: Run without prompts (requires ``org`` and ``repo``).
+        skip_key: Skip creating a new API key.
+        skip_client_config: Skip installing MCP client config.
+        dry_run: If True, return the command without executing it.
+
+    Returns:
+        A dict with ``script`` and ``args`` (the constructed argument list).
+    """
+    if not SETUP_SCRIPT.exists():
+        raise FileNotFoundError(
+            f"Setup script not found: {SETUP_SCRIPT}. "
+            "Is the ThinkStack repo checkout complete?"
+        )
+
+    if non_interactive and (not org or not repo):
+        raise ValueError("--non-interactive requires --org and --repo")
+
+    if non_interactive and not skip_client_config and not ide:
+        raise ValueError("--non-interactive requires --ide unless --skip-client-config is set")
+
+    args: list[str] = [str(SETUP_SCRIPT)]
+    if env_file:
+        args.extend(["--env-file", str(env_file)])
+    if non_interactive:
+        args.append("--non-interactive")
+    if org:
+        args.extend(["--org", org])
+    if repo:
+        args.extend(["--repo", repo])
+    if ide:
+        args.extend(["--ide", ide])
+    if openai_key:
+        args.extend(["--openai-key", openai_key])
+    if skip_key:
+        args.append("--skip-key")
+    if skip_client_config:
+        args.append("--skip-client-config")
+
+    if dry_run:
+        return {"script": str(SETUP_SCRIPT), "args": args}
+
+    result = subprocess.run(args, capture_output=False, text=True, check=False)
+    return {
+        "script": str(SETUP_SCRIPT),
+        "args": args,
+        "returncode": str(result.returncode),
+    }
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """CLI entry point used by ``thinkstack cloud setup``."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Deploy a self-hosted ThinkStack MCP server on Cloudflare Workers.",
+    )
+    parser.add_argument("--env-file", help="Path to environment file to source")
+    parser.add_argument("--org", help="ThinkStack org ID")
+    parser.add_argument("--repo", help="ThinkStack repo ID")
+    parser.add_argument("--ide", choices=["opencode", "claude", "cursor", "vscode", "stdio"],
+                        help="IDE to configure locally")
+    parser.add_argument("--openai-key", help="OpenAI API key for DRPL embedding backend")
+    parser.add_argument("--non-interactive", action="store_true",
+                        help="Run without prompts (requires --org and --repo)")
+    parser.add_argument("--skip-key", action="store_true", help="Skip creating API key")
+    parser.add_argument("--skip-client-config", action="store_true", help="Skip IDE config")
+    parser.add_argument("--dry-run", action="store_true", help="Print command instead of running")
+    args = parser.parse_args(argv)
+
+    try:
+        result = run_cloud_setup(
+            env_file=args.env_file,
+            org=args.org,
+            repo=args.repo,
+            ide=args.ide,
+            openai_key=args.openai_key,
+            non_interactive=args.non_interactive,
+            skip_key=args.skip_key,
+            skip_client_config=args.skip_client_config,
+            dry_run=args.dry_run,
+        )
+    except (FileNotFoundError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+    if args.dry_run:
+        print(" ".join(result["args"]))
+        return 0
+
+    return int(result.get("returncode", "0"))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

thinkstack_core/cloud/sync.py

@@ -0,0 +1,143 @@
+"""One-way local -> cloud reasoning sync.
+
+Pushes local .GCC commits to the ThinkStack cloud MCP worker so that
+cloud-connected agents' `thinkstack_context` sees reasoning recorded by
+local-only agents (hooks / CLI). Uses the worker's direct-POST fallback:
+POST /messages without an SSE session returns the JSON-RPC result in the
+response body.
+
+Transport hardening: if the direct-POST path fails, we open an SSE
+handshake once, capture the live endpoint URL, and retry through that
+session-bound endpoint. The SSE-derived URL is cached for all subsequent
+commits in the same sync run.
+
+Cursor: .GCC/cloud_sync_state.json {"last_pushed_index": N}. Commit ids are
+sequential hex (COMMIT_INDEX), so the cursor is a simple integer.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+_STATE_FILE = "cloud_sync_state.json"
+_USER_AGENT = "thinkstack-cloud-sync"
+
+_SSE_ENDPOINT_CACHE: str | None = None
+
+
+def _sse_handshake(sse_url: str, api_key: str, timeout: float) -> str:
+    """Open an SSE connection, capture the endpoint URL, and return it."""
+    req = urllib.request.Request(
+        sse_url,
+        headers={
+            "Accept": "text/event-stream",
+            "X-ThinkStack-Key": api_key,
+            "User-Agent": _USER_AGENT,
+        },
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        # Read line-by-line so the chunked HTTP reader does not block waiting
+        # for the next chunk after the endpoint event (the server keeps the SSE
+        # connection open for subsequent messages).
+        event_type = None
+        data = None
+        while True:
+            line = resp.readline()
+            if not line:
+                break
+            line = line.decode("utf-8", errors="replace")
+            if line.startswith("event:"):
+                event_type = line[6:].strip()
+            elif line.startswith("data:"):
+                data = line[5:].strip()
+            elif line.strip() == "":
+                if event_type == "endpoint" and data:
+                    return data
+                event_type = None
+                data = None
+    raise RuntimeError("SSE endpoint event not received")
+
+
+def sync_commits(
+    gcc_dir: Path,
+    messages_url: str,
+    api_key: str,
+    timeout: float = 30.0,
+    from_now: bool = False,
+) -> dict:
+    gcc_dir = Path(gcc_dir)
+    if not gcc_dir.is_dir():
+        raise ValueError(f"{gcc_dir} is not an initialized .GCC repository")
+    try:
+        top = int((gcc_dir / "COMMIT_INDEX").read_text(encoding="utf-8").strip() or "0")
+    except (OSError, ValueError):
+        top = 0
+
+    state_path = gcc_dir / _STATE_FILE
+    try:
+        cursor = int(json.loads(state_path.read_text(encoding="utf-8"))["last_pushed_index"])
+    except (OSError, ValueError, KeyError, json.JSONDecodeError):
+        cursor = 0
+
+    if cursor > top:
+        cursor = 0
+
+    if from_now:
+        state_path.write_text(
+            json.dumps({"last_pushed_index": top}) + "\n", encoding="utf-8"
+        )
+        return {"pushed": 0, "skipped": top, "cursor": top}
+
+    global _SSE_ENDPOINT_CACHE
+    pushed = 0
+    for index in range(cursor + 1, top + 1):
+        commit_id = format(index, "08x")
+        commit_path = gcc_dir / "commits" / f"{commit_id}.json"
+        try:
+            obj = json.loads(commit_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        message = (
+            f"[synced from local {commit_id} @ {obj.get('timestamp', 'unknown')}] "
+            f"{obj.get('message', '')}"
+        )
+        try:
+            _post_commit(messages_url, api_key, index, message, timeout)
+        except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError):
+            if _SSE_ENDPOINT_CACHE is None:
+                sse_url = messages_url.rsplit("/", 1)[0] + "/sse"
+                _SSE_ENDPOINT_CACHE = _sse_handshake(sse_url, api_key, timeout)
+            _post_commit(_SSE_ENDPOINT_CACHE, api_key, index, message, timeout)
+        pushed += 1
+        state_path.write_text(
+            json.dumps({"last_pushed_index": index}) + "\n", encoding="utf-8"
+        )
+
+    return {"pushed": pushed, "skipped": cursor, "cursor": max(cursor, top)}
+
+
+def _post_commit(
+    messages_url: str, api_key: str, rpc_id: int, message: str, timeout: float
+) -> None:
+    payload = {
+        "jsonrpc": "2.0",
+        "id": rpc_id,
+        "method": "tools/call",
+        "params": {"name": "thinkstack_commit", "arguments": {"message": message}},
+    }
+    req = urllib.request.Request(
+        messages_url,
+        data=json.dumps(payload).encode(),
+        headers={
+            "Content-Type": "application/json",
+            "X-ThinkStack-Key": api_key,
+            "User-Agent": _USER_AGENT,
+        },
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        body = json.loads(resp.read().decode())
+    if body.get("error") or body.get("result", {}).get("isError"):
+        raise RuntimeError(f"cloud commit failed: {json.dumps(body)[:200]}")