toolpipe 0.1.0__py3-none-any.whl

toolpipe/__init__.py

@@ -0,0 +1,3 @@
+"""ToolPipe: pass MCP results by reference, not through the LLM."""
+
+__version__ = "0.1.0"

toolpipe/__main__.py

@@ -0,0 +1,4 @@
+from toolpipe.cli import main
+
+if __name__ == "__main__":
+    main()

toolpipe/cli.py

@@ -0,0 +1,258 @@
+"""ToolPipe CLI: serve + local result debugging."""
+
+import argparse
+import asyncio
+import contextlib
+import json
+import logging
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from toolpipe.config import load_config
+from toolpipe.errors import ConfigurationError, ToolPipeError
+from toolpipe.models import ToolPipeConfig
+from toolpipe.results.cleanup import periodic_cleanup, run_startup_cleanup
+from toolpipe.results.manager import ResultManager
+from toolpipe.server.app import create_app
+from toolpipe.utils.sizes import format_bytes
+from toolpipe.utils.time import parse_iso, utc_now
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="toolpipe", description="Pass MCP results by reference, not through the LLM."
+    )
+    sub = p.add_subparsers(dest="command")
+    serve = sub.add_parser("serve", help="Run ToolPipe MCP server (STDIO).")
+    serve.add_argument("--config", default=None)
+    serve.add_argument("--log-level", default="WARNING")
+    serve.add_argument("--import-mcp-json", default=None)
+    serve.add_argument("--import-codex-toml", default=None)
+    serve.add_argument("--no-discover", action="store_true")
+    serve.add_argument("--no-dynamic-servers", action="store_true")
+
+    inspect_p = sub.add_parser("inspect", help="Show a stored result (local debugging).")
+    inspect_p.add_argument("ref")
+    inspect_p.add_argument("--config", default="toolpipe.toml")
+
+    results_p = sub.add_parser("results", help="List stored results.")
+    results_p.add_argument("--config", default="toolpipe.toml")
+
+    stats_p = sub.add_parser("stats", help="Show result store metrics.")
+    stats_p.add_argument("--config", default="toolpipe.toml")
+
+    clean_p = sub.add_parser("clean", help="Remove expired results.")
+    clean_p.add_argument("--config", default="toolpipe.toml")
+    return p
+
+
+def _configure_logging(level: str) -> None:
+    logging.basicConfig(stream=sys.stderr, level=getattr(logging, level.upper(), logging.WARNING))
+
+
+def _open_manager(config_path: str) -> tuple[ToolPipeConfig, ResultManager]:
+    try:
+        config = load_config(config_path)
+    except ConfigurationError as e:
+        print(f"toolpipe: configuration error: {e}", file=sys.stderr)
+        raise SystemExit(2)
+    return config, ResultManager.from_storage_dir(config.results.storage_dir, config.results)
+
+
+def _load_serve_config(args: argparse.Namespace) -> tuple[ToolPipeConfig, list]:
+    """Build the serve config.
+
+    Returns (config, discovered): TOML + explicit imports are pre-approved;
+    auto-discovered entries stay pending until first-use consent.
+    """
+    from toolpipe import discovery
+    from toolpipe.config import merge_server_maps, servers_from_discovered
+    from toolpipe.models import ServerConfig
+
+    if args.config is not None:
+        base = load_config(args.config)
+    elif Path("toolpipe.toml").exists():
+        base = load_config("toolpipe.toml")
+    else:
+        base = ToolPipeConfig()
+
+    imported: dict = {}
+    if args.import_mcp_json is not None:
+        try:
+            text = Path(args.import_mcp_json).read_text()
+        except OSError as e:
+            raise ConfigurationError(f"Cannot read {args.import_mcp_json}: {e}")
+        try:
+            entries = discovery.parse_mcp_json(text, args.import_mcp_json)
+        except ValueError as e:
+            raise ConfigurationError(f"Invalid MCP JSON {args.import_mcp_json}: {e}")
+        imported.update(servers_from_discovered(entries, strict=True, source_label="import"))
+    if args.import_codex_toml is not None:
+        try:
+            text = Path(args.import_codex_toml).read_text()
+        except OSError as e:
+            raise ConfigurationError(f"Cannot read {args.import_codex_toml}: {e}")
+        try:
+            entries = discovery.parse_codex_toml(text, args.import_codex_toml)
+        except ValueError as e:
+            raise ConfigurationError(f"Invalid Codex TOML {args.import_codex_toml}: {e}")
+        imported.update(servers_from_discovered(entries, strict=True, source_label="import"))
+
+    discovered: dict = {}
+    if not args.no_discover:
+        discovered = servers_from_discovered(
+            discovery.discover("."), strict=False, source_label="discovery"
+        )
+
+    # Precedence: TOML < explicit imports. Discovered entries are returned
+    # separately as pending (consent on first use), not merged as active.
+    # Persisted scope files layer around this chain (see ConsentStore).
+    servers = merge_server_maps(base.servers, imported)
+    pending = [
+        ServerConfig(name=name, command=cfg.command, args=cfg.args, url=cfg.url, env=cfg.env)
+        for name, cfg in discovered.items()
+        if name not in servers
+    ]
+    return (
+        ToolPipeConfig(name=base.name, results=base.results, servers=servers),
+        pending,
+    )
+
+
+def _rel(iso: str, now: datetime) -> str:
+    seconds = max(0, int((now - parse_iso(iso)).total_seconds()))
+    return _fmt_duration(seconds)
+
+
+def _rel_remaining(iso: str, now: datetime) -> str:
+    seconds = max(0, int((parse_iso(iso) - now).total_seconds()))
+    return _fmt_duration(seconds)
+
+
+def _fmt_duration(seconds: int) -> str:
+    if seconds < 60:
+        return f"{seconds}s"
+    if seconds < 3600:
+        return f"{seconds // 60}m"
+    if seconds < 86400:
+        return f"{seconds // 3600}h"
+    return f"{seconds // 86400}d"
+
+
+def _serve(args: argparse.Namespace) -> None:
+    _configure_logging(args.log_level)
+    try:
+        config, discovered = _load_serve_config(args)
+
+        async def _run() -> None:
+            app = await create_app(
+                config, discovered=discovered, dynamic_servers=not args.no_dynamic_servers
+            )
+            if not config.servers:
+                print(
+                    "toolpipe: no downstream servers configured — "
+                    "only control tools are exposed. Add servers via "
+                    "toolpipe_add_server, --config, or --import-mcp-json.",
+                    file=sys.stderr,
+                )
+            run_startup_cleanup(app.manager)
+            cleanup_task = asyncio.create_task(periodic_cleanup(app.manager))
+            try:
+                # STDIO is the protocol channel; logs go to stderr only.
+                await app.mcp.run_stdio_async()
+            finally:
+                cleanup_task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await cleanup_task
+                app.close()
+
+        asyncio.run(_run())
+    except ConfigurationError as e:
+        print(f"toolpipe: configuration error: {e}", file=sys.stderr)
+        raise SystemExit(2)
+
+
+def _inspect(config_path: str, ref: str) -> None:
+    _, manager = _open_manager(config_path)
+    try:
+        info = manager.inspect(ref)
+    except ToolPipeError as e:
+        print(f"toolpipe: {e}", file=sys.stderr)
+        raise SystemExit(1)
+    finally:
+        manager.close()
+    print(
+        json.dumps(
+            {
+                "ref": info.ref,
+                "content_type": info.content_type,
+                "size_bytes": info.size_bytes,
+                "source_tool": info.source_tool,
+                "created_at": info.created_at,
+                "expires_at": info.expires_at,
+                "preview": json.loads(info.preview_json) if info.preview_json else None,
+            },
+            indent=2,
+        )
+    )
+
+
+def _results(config_path: str) -> None:
+    _, manager = _open_manager(config_path)
+    try:
+        rows = manager.list_results()
+    finally:
+        manager.close()
+    now = utc_now()
+    print(f"{'REF':<24} {'TYPE':<18} {'SIZE':<10} {'AGE':<6} {'EXPIRES'}")
+    for row in rows:
+        print(
+            f"{row.ref:<24} {row.content_type:<18} {format_bytes(row.size_bytes):<10} "
+            f"{_rel(row.created_at, now):<6} {_rel_remaining(row.expires_at, now)}"
+        )
+
+
+def _stats(config_path: str) -> None:
+    _, manager = _open_manager(config_path)
+    try:
+        stats = manager.stats()
+    finally:
+        manager.close()
+    print(f"Proxied tool calls:          {stats['proxied_tool_calls']}")
+    print(f"Virtualized results:         {stats['virtualized_results']}")
+    print(f"Bytes virtualized:           {format_bytes(stats['virtualized_bytes'])}")
+    print(f"Bytes returned as refs:      {format_bytes(stats['reference_response_bytes'])}")
+    print(f"Estimated bytes avoided:     {format_bytes(stats['estimated_bytes_avoided'])}")
+    print(f"Pipe calls:                  {stats['pipe_calls']}")
+    print(f"Stored results:              {stats['stored_results']}")
+    print(f"Current storage:             {format_bytes(stats['current_storage_bytes'])}")
+
+
+def _clean(config_path: str) -> None:
+    _, manager = _open_manager(config_path)
+    try:
+        count = manager.cleanup_expired()
+    finally:
+        manager.close()
+    print(f"Removed {count} expired result(s).")
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = build_parser().parse_args(argv)
+    if args.command is None:
+        build_parser().print_help()
+    elif args.command == "serve":
+        _serve(args)
+    elif args.command == "inspect":
+        _inspect(args.config, args.ref)
+    elif args.command == "results":
+        _results(args.config)
+    elif args.command == "stats":
+        _stats(args.config)
+    elif args.command == "clean":
+        _clean(args.config)
+
+
+if __name__ == "__main__":
+    main()

toolpipe/config.py

@@ -0,0 +1,163 @@
+"""TOML configuration loading with ${VAR} environment substitution."""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import tomllib
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from toolpipe.errors import ConfigurationError
+from toolpipe.models import ResultSettings, ServerConfig, ToolPipeConfig
+
+if TYPE_CHECKING:
+    from toolpipe.discovery import DiscoveredServer
+
+logger = logging.getLogger(__name__)
+
+_ENV_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
+
+
+def _substitute_env(value: str) -> str:
+    def repl(match: re.Match[str]) -> str:
+        var = match.group(1)
+        if var not in os.environ:
+            raise ConfigurationError(f"Environment variable {var} is not set.")
+        return os.environ[var]
+
+    return _ENV_PATTERN.sub(repl, value)
+
+
+def resolve_env_vars(env: dict[str, str]) -> dict[str, str]:
+    """Resolve ${VAR} references now (fail fast on missing variables)."""
+    return {k: _substitute_env(v) for k, v in env.items()}
+
+
+def _substitute_deep(value: object) -> object:
+    if isinstance(value, str):
+        return _substitute_env(value)
+    if isinstance(value, dict):
+        return {k: _substitute_deep(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_substitute_deep(v) for v in value]
+    return value
+
+
+def load_config(path: str | Path) -> ToolPipeConfig:
+    """Load and validate a toolpipe.toml file."""
+    path = Path(path)
+    try:
+        raw = tomllib.loads(path.read_text())
+    except FileNotFoundError:
+        raise ConfigurationError(f"Config file not found: {path}")
+    except tomllib.TOMLDecodeError as e:
+        raise ConfigurationError(f"Invalid TOML in {path}: {e}")
+
+    raw = _substitute_deep(raw)
+    assert isinstance(raw, dict)
+
+    toolpipe_raw = raw.get("toolpipe", {})
+    if not isinstance(toolpipe_raw, dict):
+        raise ConfigurationError("[toolpipe] must be a table.")
+    name = toolpipe_raw.get("name", "ToolPipe")
+    if not isinstance(name, str) or not name:
+        raise ConfigurationError("[toolpipe].name must be a non-empty string.")
+
+    results_raw = raw.get("results", {})
+    if not isinstance(results_raw, dict):
+        raise ConfigurationError("[results] must be a table.")
+    unknown = set(results_raw) - set(ResultSettings.__dataclass_fields__)
+    if unknown:
+        raise ConfigurationError(f"Unknown [results] settings: {sorted(unknown)}.")
+    try:
+        results = ResultSettings(**results_raw)
+    except TypeError as e:
+        raise ConfigurationError(f"Invalid [results] setting: {e}")
+    for field in (
+        "inline_max_bytes",
+        "preview_max_bytes",
+        "read_max_bytes",
+        "max_result_bytes",
+        "max_store_bytes",
+        "ttl_seconds",
+    ):
+        if getattr(results, field) < 0:
+            raise ConfigurationError(f"[results].{field} must be >= 0.")
+    if not results.storage_dir:
+        raise ConfigurationError("[results].storage_dir must be a non-empty string.")
+
+    servers_raw = raw.get("servers", {})
+    if not isinstance(servers_raw, dict):
+        raise ConfigurationError("[servers] must be a table.")
+    servers: dict[str, ServerConfig] = {}
+    for server_name, spec in servers_raw.items():
+        if not isinstance(spec, dict):
+            raise ConfigurationError(f"[servers.{server_name}] must be a table.")
+        command = spec.get("command")
+        url = spec.get("url")
+        if command is None and url is None:
+            raise ConfigurationError(f"[servers.{server_name}] needs either 'command' or 'url'.")
+        if command is not None and url is not None:
+            raise ConfigurationError(
+                f"[servers.{server_name}] cannot set both 'command' and 'url'."
+            )
+        if command is not None and not isinstance(command, str):
+            raise ConfigurationError(f"[servers.{server_name}.command] must be a string.")
+        if url is not None and not isinstance(url, str):
+            raise ConfigurationError(f"[servers.{server_name}.url] must be a string.")
+        args = spec.get("args", [])
+        env = spec.get("env", {})
+        if not isinstance(args, list) or not all(isinstance(a, str) for a in args):
+            raise ConfigurationError(f"[servers.{server_name}.args] must be a list of strings.")
+        if not isinstance(env, dict) or not all(
+            isinstance(k, str) and isinstance(v, str) for k, v in env.items()
+        ):
+            raise ConfigurationError(f"[servers.{server_name}.env] must be string key/values.")
+        servers[server_name] = ServerConfig(
+            name=server_name, command=command, args=args, url=url, env=dict(env)
+        )
+
+    return ToolPipeConfig(name=name, results=results, servers=servers)
+
+
+def servers_from_discovered(
+    entries: list[DiscoveredServer], *, strict: bool, source_label: str = "discovery"
+) -> dict[str, ServerConfig]:
+    """Convert discovery entries to ServerConfigs.
+
+    Applies ${VAR} substitution. Strict mode (explicit --import-*) raises on
+    bad entries; auto-discovery skips them with a warning instead.
+    """
+    servers: dict[str, ServerConfig] = {}
+    for entry in entries:
+        try:
+            env = {k: _substitute_env(v) for k, v in entry.env.items()}
+        except ConfigurationError as e:
+            if strict:
+                raise
+            logger.warning("%s skipped server '%s': %s", source_label, entry.name, e)
+            continue
+        if entry.command is None and entry.url is None:
+            err = ConfigurationError(f"Server '{entry.name}' needs 'command' or 'url'.")
+            if strict:
+                raise err
+            logger.warning("%s skipped server '%s': %s", source_label, entry.name, err)
+            continue
+        servers[entry.name] = ServerConfig(
+            name=entry.name,
+            command=entry.command,
+            args=list(entry.args),
+            url=entry.url,
+            env=env,
+        )
+    return servers
+
+
+def merge_server_maps(*maps: dict[str, ServerConfig]) -> dict[str, ServerConfig]:
+    """Merge server maps; later maps win on name collisions."""
+    merged: dict[str, ServerConfig] = {}
+    for mapping in maps:
+        merged.update(mapping)
+    return merged

toolpipe/constants.py

@@ -0,0 +1,11 @@
+"""Shared constants."""
+
+CONTROL_TAG = "toolpipe-control"
+RESERVED_NAMESPACE = "toolpipe"
+REF_PREFIX = "res_"
+
+INSTRUCTIONS = (
+    "Large downstream results may be returned as ToolPipe result references. "
+    "Use toolpipe_inspect_result, toolpipe_select_result, or "
+    "toolpipe_pipe_result instead of requesting the full result."
+)