columna-server 0.1.0__py3-none-any.whl

columna_server/__init__.py

@@ -0,0 +1,2 @@
+"""Columna MCP server — a library of Manifolds exposed to AI agents over MCP (ADR-032 D8)."""
+__version__ = "0.1.0"

columna_server/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

columna_server/agent/__init__.py

@@ -0,0 +1,10 @@
+"""columna_server.agent — the natural-language query agent (a true MCP client; ADR-032 D8)."""
+from .conversation import AGENT, ENGINE, USER, Turn
+from .loop import Agent, load_system_prompt, prompt_example_queries, render_manifold_context
+from .providers import (AnthropicProvider, Provider, ProviderUnavailable, ScriptedProvider,
+                        make_provider)
+
+__all__ = ["Turn", "USER", "AGENT", "ENGINE", "Agent", "load_system_prompt",
+           "prompt_example_queries", "render_manifold_context",
+           "Provider", "ProviderUnavailable", "ScriptedProvider", "AnthropicProvider",
+           "make_provider"]

columna_server/agent/conversation.py

@@ -0,0 +1,14 @@
+"""columna_server.agent.conversation — the shared turn model."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+USER = "user"      # a human message
+AGENT = "agent"    # the agent's proposed line (QUERY:/ASK:)
+ENGINE = "engine"  # a compact wire summary the system appends after each query
+
+
+@dataclass(frozen=True)
+class Turn:
+    role: str      # USER | AGENT | ENGINE
+    text: str

columna_server/agent/loop.py

@@ -0,0 +1,220 @@
+"""
+columna_server.agent.loop — the agent conversation loop (WP-2.4 architecture #2).
+
+user NL → provider proposes ONE Frame-QL query (or an ASK) → the MCP `query` tool → the outcome
+routes the turn. The SYSTEM presents every server reply to the human, rendering wire values verbatim
+(so the agent never emits a number — grounding is structural). A clarify is relayed to the human and
+resolved only by their explicit choice (never auto-picked); a refuse permits ONE reformulation.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+
+from .conversation import AGENT, ENGINE, USER, Turn
+from .providers import Provider
+
+_PROMPT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "system_prompt.md")
+
+
+def load_system_prompt() -> str:
+    with open(_PROMPT_PATH, encoding="utf-8") as f:
+        return f.read()
+
+
+def prompt_example_queries() -> list[str]:
+    """Every `QUERY:` example line in the system prompt (for the drift-guard parse test)."""
+    out = []
+    for line in load_system_prompt().splitlines():
+        s = line.strip().lstrip("- ").strip()
+        if s.startswith("QUERY:"):
+            out.append(s[len("QUERY:"):].strip())
+    return out
+
+
+def render_manifold_context(describe: dict) -> str:
+    lines = [f"Manifold: {describe['manifold_id']}"]
+    lines.append("Universes (a measure lives in exactly one):")
+    for u in describe["universes"]:
+        pred = f"  WHERE {u['predicate']}" if u["predicate"] else ""
+        lines.append(f"  - {u['name']}: over {', '.join(u['base_dimensions'])}{pred}")
+    lines.append("Dimension levels (anchors you may address at):")
+    lines.append("  " + ", ".join(sorted(d["level"] for d in describe["dimensions"])))
+    if describe["edges"]:
+        lines.append("Functional edges (finer -> coarser, along a lineage):")
+        for e in describe["edges"]:
+            lines.append(f"  - {e['frm']} -> {e['to']} ({e['lineage']})")
+    lines.append("Measures (name — family members — universe):")
+    for m in describe["measures"]:
+        fam = "/".join(m["family"]) if m["family"] else "(single)"
+        lines.append(f"  - {m['name']}  [{fam}]  on {m['universe']}")
+    return "\n".join(lines)
+
+
+_NUM = re.compile(r"-?\d+(?:\.\d+)?")
+
+
+class Agent:
+    """Drives one conversation. `run_turn(user_msg)` returns the human-facing replies for the turn.
+
+    Not an MCP server itself — it holds an `MCPServerConnection` and calls tools across the wire.
+    """
+
+    def __init__(self, conn, provider: Provider, describe: dict):
+        self.conn = conn
+        self.provider = provider
+        self.system = load_system_prompt().replace("{MANIFOLD_CONTEXT}", render_manifold_context(describe))
+        self.history: list[Turn] = []
+        self._pending: tuple[list[dict], str] | None = None   # (alternatives, frameql) awaiting a choice
+
+    # ---- turn routing -------------------------------------------------------
+    async def run_turn(self, user_msg: str) -> list[str]:
+        replies: list[str] = []
+
+        # (A) a clarify is open — this message is the human's CHOICE (never auto-picked)
+        if self._pending is not None:
+            alts, frameql = self._pending
+            self._pending = None
+            idx = self._resolve_choice(user_msg, alts)
+            self.history.append(Turn(USER, user_msg))
+            if idx is None:
+                replies.append("I didn't catch which option — reply with its number, or ask something new.")
+                # fall through and treat this message as a fresh request
+            else:
+                universe = (alts[idx].get("apply") or {}).get("universe")
+                wire = await self.conn.query(frameql, universe=universe)
+                self.history.append(Turn(ENGINE, self._engine_summary(wire)))
+                replies.append(self._present(wire, chosen_universe=universe))
+                return replies
+
+        # (B) fresh proposal — allow exactly ONE reformulation after a refuse/error
+        reformulations_left = 1
+        current = user_msg
+        while True:
+            line = self.provider.propose(self.system, list(self.history), current)
+            self.history.append(Turn(USER, current))
+            self.history.append(Turn(AGENT, line))
+            kind, body = _parse_line(line)
+
+            if kind == "ask":
+                replies.append(body)
+                return replies
+            if kind == "invalid":
+                replies.append("(couldn't read that as a Frame-QL query — try rephrasing your question)")
+                return replies
+
+            wire = await self.conn.query(body)
+            self.history.append(Turn(ENGINE, self._engine_summary(wire)))
+            replies.append(self._present(wire))
+            outcome = wire["outcome"]
+
+            if outcome == "clarify":
+                self._pending = (self._alternatives(wire), body)
+                return replies
+            if outcome in ("refuse", "error") and reformulations_left > 0:
+                reformulations_left -= 1
+                current = ("[reformulate] The previous query was not answerable (see the engine "
+                           "note). If a reformulation addresses the reason, propose exactly one; "
+                           "otherwise ask the human or stop.")
+                continue
+            return replies
+
+    # ---- presentation (verbatim from the wire; the agent adds no numbers) ---
+    def _present(self, wire: dict, chosen_universe: str | None = None) -> str:
+        outcome = wire["outcome"]
+        if outcome in ("serve", "disclose"):
+            parts = []
+            if chosen_universe:
+                parts.append(f"[over universe '{chosen_universe}']")
+            for col in wire["columns"]:
+                if col["status"] != "served":
+                    continue
+                parts.append(f"{col['name']}: {_render_values(col)}")
+                for d in col["disclosures"]:
+                    parts.append(f"  · disclosure [{d['code']}, {d['materiality']}]: {d['detail']}")
+            for d in wire["frame"]["disclosures"]:
+                parts.append(f"· disclosure [{d['code']}, {d['materiality']}]: {d['detail']}")
+            head = "Here is the answer:" if outcome == "serve" else "Answer (with caveats):"
+            return head + "\n" + "\n".join(parts)
+
+        if outcome == "clarify":
+            col = next(c for c in wire["columns"] if c["status"] == "clarify")
+            nr = col["no_result"]
+            lines = [f"That question is ambiguous: {nr['detail']}", "Please choose:"]
+            for i, a in enumerate(self._alternatives(wire), 1):
+                lines.append(f"  [{i}] {a['description']}")
+            lines.append("Reply with the number of your choice.")
+            return "\n".join(lines)
+
+        # refuse / error
+        col = next((c for c in wire["columns"] if c["status"] in ("refuse", "error")), None)
+        if col is not None:
+            nr = col["no_result"]
+            verb = "can't be answered" if outcome == "refuse" else "isn't a valid query"
+            return f"That {verb}: {nr['detail']}"
+        err = wire.get("error", {})
+        return f"That isn't a valid query: {err.get('detail', 'unknown error')}"
+
+    # ---- engine note (compact context for the model's NEXT proposal) --------
+    def _engine_summary(self, wire: dict) -> str:
+        outcome = wire["outcome"]
+        if outcome in ("serve", "disclose"):
+            cols = ", ".join(c["name"] for c in wire["columns"] if c["status"] == "served")
+            mats = [d["code"] for c in wire["columns"] for d in c["disclosures"] if d["materiality"] == "material"]
+            mats += [d["code"] for d in wire["frame"]["disclosures"] if d["materiality"] == "material"]
+            extra = f"; material disclosures: {mats}" if mats else ""
+            return f"engine: outcome={outcome}; served columns [{cols}]{extra}. (values shown to the human)"
+        col = next((c for c in wire["columns"] if c["status"] in ("clarify", "refuse", "error")), None)
+        nr = col["no_result"] if col else {}
+        alts = [a.get("token", a.get("description", "")) for a in self._alternatives(wire)]
+        tail = f"; alternatives: {alts}" if alts else ""
+        return (f"engine: outcome={outcome}; reason={nr.get('reason')}; detail={nr.get('detail')}"
+                f"{tail}. (shown to the human; you do not restate it)")
+
+    # ---- helpers ------------------------------------------------------------
+    @staticmethod
+    def _alternatives(wire: dict) -> list[dict]:
+        for c in wire["columns"]:
+            if c.get("no_result") and c["no_result"].get("alternatives"):
+                return c["no_result"]["alternatives"]
+        return []
+
+    @staticmethod
+    def _resolve_choice(user_msg: str, alts: list[dict]) -> int | None:
+        s = user_msg.strip().lower()
+        m = re.search(r"\d+", s)
+        if m:
+            i = int(m.group()) - 1
+            if 0 <= i < len(alts):
+                return i
+        for i, a in enumerate(alts):
+            uni = (a.get("apply") or {}).get("universe", "")
+            if uni and uni.lower() in s:
+                return i
+        return None
+
+
+def _render_values(col: dict) -> str:
+    if "value" in col:
+        return _s(col["value"])
+    rows = col.get("values", [])
+    out = []
+    for r in rows:
+        dims = " ".join(f"{k}={_s(v)}" for k, v in r.items() if k != "value")
+        out.append(f"{dims}={_s(r['value'])}" if dims else _s(r["value"]))
+    return "; ".join(out) if out else "(no rows)"
+
+
+def _s(v) -> str:
+    """Verbatim stringification — no rounding, so grounding matches strictly."""
+    return json.dumps(v) if isinstance(v, str) else str(v)
+
+
+def _parse_line(line: str) -> tuple[str, str]:
+    s = line.strip()
+    if s.startswith("QUERY:"):
+        return "query", s[len("QUERY:"):].strip()
+    if s.startswith("ASK:"):
+        return "ask", s[len("ASK:"):].strip()
+    return "invalid", s

columna_server/agent/mcp_client.py

@@ -0,0 +1,80 @@
+"""
+columna_server.agent.mcp_client — the agent's TRUE MCP client (WP-2.4 architecture #1).
+
+The agent answers ONLY by spawning `columna-server mcp/demo` over stdio and speaking the MCP
+protocol. This module imports the MCP client SDK and nothing from columna_core — the engine is
+reached across the protocol boundary, never in-process. Bypassing this boundary fails the WP.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from contextlib import asynccontextmanager
+
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+
+
+def _server_args(manifolds: str | None) -> list[str]:
+    # Spawn the server as its own process, exactly as any external agent would.
+    if manifolds:
+        return ["-m", "columna_server", "mcp", "--manifolds", manifolds]
+    return ["-m", "columna_server", "demo"]           # the packaged demo, zero path args
+
+
+class MCPServerConnection:
+    """A live MCP session to a columna server. Read-only tool calls; results are the wire JSON."""
+
+    def __init__(self, session: ClientSession):
+        self._s = session
+        self.manifold_id: str | None = None
+
+    async def _call(self, name: str, **args) -> dict:
+        res = await self._s.call_tool(name, args)
+        if res.isError:
+            text = res.content[0].text if res.content else "tool error"
+            raise RuntimeError(f"MCP tool '{name}' error: {text}")
+        return json.loads(res.content[0].text)
+
+    async def tool_names(self) -> list[str]:
+        return sorted(t.name for t in (await self._s.list_tools()).tools)
+
+    async def list_manifolds(self) -> dict:
+        return await self._call("list_manifolds")
+
+    async def describe_manifold(self, manifold_id: str) -> dict:
+        return await self._call("describe_manifold", manifold_id=manifold_id)
+
+    async def describe_measure(self, manifold_id: str, measure: str) -> dict:
+        return await self._call("describe_measure", manifold_id=manifold_id, measure=measure)
+
+    async def query(self, frameql: str, universe: str | None = None) -> dict:
+        args = {"manifold_id": self.manifold_id, "frameql": frameql}
+        if universe:
+            args["universe"] = universe
+        return await self._call("query", **args)
+
+    async def explain(self, frameql: str, universe: str | None = None) -> dict:
+        args = {"manifold_id": self.manifold_id, "frameql": frameql}
+        if universe:
+            args["universe"] = universe
+        return await self._call("explain", **args)
+
+
+@asynccontextmanager
+async def connect(manifolds: str | None = None):
+    """Spawn a columna server over stdio and yield a connected MCPServerConnection.
+
+    `manifolds=None` connects to the packaged demo (`columna-server demo`); a path connects to
+    `columna-server mcp --manifolds <path>`. The first hosted manifold is selected by default.
+    """
+    params = StdioServerParameters(command=sys.executable, args=_server_args(manifolds),
+                                   env=dict(os.environ))
+    async with stdio_client(params) as (read, write):
+        async with ClientSession(read, write) as session:
+            await session.initialize()
+            conn = MCPServerConnection(session)
+            listed = await conn.list_manifolds()
+            conn.manifold_id = listed["manifolds"][0]["manifold_id"]
+            yield conn

columna_server/agent/providers.py

@@ -0,0 +1,115 @@
+"""
+columna_server.agent.providers — the tiny provider layer (WP-2.4 ruling 3-B).
+
+`propose(system, history, user_msg) -> str` returns the model's next line — either `QUERY: <frameql>`
+or `ASK: <question>`. The provider ONLY produces text; it never touches MCP or the engine. Two
+implementations: `anthropic` (default, BYO key) and `scripted` (deterministic, for tests/demos).
+"""
+from __future__ import annotations
+
+import os
+from typing import Protocol, runtime_checkable
+
+from .conversation import AGENT, ENGINE, Turn
+
+DEFAULT_MODEL = "claude-opus-4-8"
+MODEL_ENV = "COLUMNA_AGENT_MODEL"
+KEY_ENV = "ANTHROPIC_API_KEY"
+
+
+class ProviderUnavailable(RuntimeError):
+    """The requested provider cannot run (e.g. no API key)."""
+
+
+@runtime_checkable
+class Provider(Protocol):
+    name: str
+
+    def propose(self, system: str, history: list[Turn], user_msg: str) -> str:
+        """Return the next agent line: 'QUERY: <frameql>' or 'ASK: <question>'."""
+        ...
+
+
+class ScriptedProvider:
+    """Deterministic provider: replays a fixed list of lines, one per call. For tests and demos —
+    no network, no key. Raises if the script runs dry (a sign the conversation went further than
+    the test scripted)."""
+
+    name = "scripted"
+
+    def __init__(self, script: list[str]):
+        self._script = list(script)
+        self._i = 0
+
+    def propose(self, system: str, history: list[Turn], user_msg: str) -> str:
+        if self._i >= len(self._script):
+            raise ProviderUnavailable("scripted provider exhausted — the agent asked for more "
+                                      "proposals than the script supplies")
+        line = self._script[self._i]
+        self._i += 1
+        return line
+
+
+class AnthropicProvider:
+    """Default provider — one `messages.create` per turn via the official anthropic SDK. Model from
+    COLUMNA_AGENT_MODEL (default claude-opus-4-8); key from ANTHROPIC_API_KEY. Deps live in the
+    `[agent]` extra; no key -> a clear error pointing at the packaged demo."""
+
+    name = "anthropic"
+
+    def __init__(self, model: str | None = None):
+        self.model = model or os.environ.get(MODEL_ENV, DEFAULT_MODEL)
+        if not os.environ.get(KEY_ENV):
+            raise ProviderUnavailable(
+                f"{KEY_ENV} is not set. Set it to use the natural-language agent, or try the "
+                f"no-key packaged demo: `columna-server demo --play`."
+            )
+        try:
+            import anthropic
+        except ModuleNotFoundError as e:
+            raise ProviderUnavailable(
+                "the anthropic SDK is not installed — install the agent extra: "
+                "`pip install -e \"packages/columna-server[agent]\"`."
+            ) from e
+        self._client = anthropic.Anthropic()
+
+    def propose(self, system: str, history: list[Turn], user_msg: str) -> str:
+        # Render the conversation as alternating user/assistant messages. The engine notes and the
+        # human turns become `user` content (context for the model); the agent's own prior lines
+        # become `assistant` content. The system prompt is passed separately.
+        messages = []
+        for t in history:
+            if t.role == AGENT:
+                messages.append({"role": "assistant", "content": t.text})
+            else:  # USER or ENGINE — both are context the model reads
+                prefix = "[engine] " if t.role == ENGINE else ""
+                messages.append({"role": "user", "content": prefix + t.text})
+        messages.append({"role": "user", "content": user_msg})
+
+        resp = self._client.messages.create(
+            model=self.model, max_tokens=512, system=system, messages=_coalesce(messages),
+        )
+        return "".join(b.text for b in resp.content if b.type == "text").strip()
+
+
+def _coalesce(messages: list[dict]) -> list[dict]:
+    """The Messages API requires the first message to be `user` and forbids empty history. Merge
+    consecutive same-role messages so a run of engine+user notes stays one turn."""
+    out: list[dict] = []
+    for m in messages:
+        if out and out[-1]["role"] == m["role"]:
+            out[-1]["content"] = out[-1]["content"] + "\n" + m["content"]
+        else:
+            out.append(dict(m))
+    if not out or out[0]["role"] != "user":
+        out.insert(0, {"role": "user", "content": "(begin)"})
+    return out
+
+
+def make_provider(name: str, model: str | None = None) -> Provider:
+    if name == "scripted":
+        raise ProviderUnavailable("the scripted provider requires an explicit script; "
+                                  "construct ScriptedProvider(script) directly")
+    if name == "anthropic":
+        return AnthropicProvider(model=model)
+    raise ProviderUnavailable(f"unknown provider '{name}' (have: anthropic, scripted)")

columna_server/agent/runner.py

@@ -0,0 +1,39 @@
+"""columna_server.agent.runner — the `columna-server agent` REPL."""
+from __future__ import annotations
+
+import asyncio
+import sys
+
+from .loop import Agent
+from .mcp_client import connect
+from .providers import ProviderUnavailable, make_provider
+
+
+async def _run(manifolds, provider):
+    async with connect(manifolds) as conn:
+        describe = await conn.describe_manifold(conn.manifold_id)
+        agent = Agent(conn, provider, describe)
+        print(f"Columna agent — manifold '{conn.manifold_id}', provider '{provider.name}'. "
+              f"Ask a question; Ctrl-D to exit.", file=sys.stderr)
+        while True:
+            try:
+                line = input("you> ")
+            except EOFError:
+                print(file=sys.stderr)
+                break
+            if not line.strip():
+                continue
+            for reply in await agent.run_turn(line):
+                print(reply)
+                print()
+
+
+def run_agent_cli(manifolds: str | None = None, provider_name: str = "anthropic",
+                  model: str | None = None) -> int:
+    try:
+        provider = make_provider(provider_name, model=model)
+    except ProviderUnavailable as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+    asyncio.run(_run(manifolds, provider))
+    return 0

columna_server/agent/system_prompt.md

@@ -0,0 +1,66 @@
+You are the Columna query agent. You turn a person's natural-language question into ONE Frame-QL
+query, hand it to the Columna server (the analytic engine), and let the system relay what the engine
+returns. You are a proposer, not an authority: the engine decides whether a query is answerable, and
+the human decides between genuine alternatives. Your job is to propose well.
+
+## Your role boundary
+
+You never write result numbers or answer prose. The server's reply — values, disclosures,
+alternatives, refusal reasons — is presented to the human by the system, verbatim from the wire. You
+only propose queries and ask questions. Do not restate, summarize, round, or interpret results.
+
+## What you may emit
+
+On every turn you emit EXACTLY ONE of:
+1. A Frame-QL query line, prefixed literally with `QUERY: ` — nothing else on the line.
+2. A short question back to the human, prefixed literally with `ASK: ` — only when the request is
+   too ambiguous to form any reasonable query, so you need the human to say what they want.
+
+Never emit both. Never emit prose outside these two prefixes. Never wrap the query in backticks.
+
+## Frame-QL (the only query language — no SQL, ever)
+
+A query is `<columns> @ <anchor>`:
+- `<columns>` is one or more comma-separated columns; each is `name: expression` or a bare
+  `expression` (which names itself). An expression is measure arithmetic over the manifold's
+  measures, e.g. `revenue`, `revenue / orders`, `level.sum`, `level.last`.
+- `<anchor>` is one or more comma-separated dimension levels, e.g. `store`, `store, day`,
+  `cal.month`, `region`.
+- Commas inside function calls (e.g. `lag(revenue.sum, n=1)`) are not column separators.
+
+Examples:
+- QUERY: revenue @ region
+- QUERY: rate: revenue / level.last @ store, day
+- QUERY: rev: revenue, inv: level.last @ store, day
+- QUERY: m: lag(revenue.sum, n=1) @ cal.month
+
+Only use measure, derived, dimension, and universe names that appear in the manifold description
+below. Do not invent columns, operators, or anchors. If a measure has a family (e.g. `sum`,
+`last`), address a member as `measure.member`.
+
+## What the engine sends back (your context, not the human's answer)
+
+After each query the system adds an `engine` note to the conversation summarizing the outcome:
+`serve` / `disclose` (the values were shown), `clarify` (the population is ambiguous — the human is
+being shown the candidate alternatives and will choose; you do not choose), `refuse` / `error` (the
+reason was shown). Use these notes only to decide your NEXT proposal — never to write an answer.
+
+## Rules you must follow
+
+- GROUNDING: you never emit a number, so you can never state a wrong one. If the human asks for a
+  figure you do not have (e.g. a total across rows), propose a query that measures it — never do the
+  arithmetic yourself.
+- POPULATION: a measure belongs to one universe. A ratio of measures from different universes has no
+  single population; the engine will clarify and the human will pick. Do not pre-pick a population.
+- CLARIFY IS THE HUMAN'S CALL: on a clarify, do not re-propose the same ratio with a population
+  guessed in. The system applies the human's chosen alternative for them.
+- ONE REFORMULATION AFTER A REFUSE: when the engine refuses (e.g. a measure addressed outside its
+  universe), you MAY propose exactly ONE reformulated query that addresses the stated reason (for a
+  cross-universe ratio, that is usually the two measures as separate columns). If the reformulation
+  also fails, stop — do not keep retrying.
+- HONEST NAMES: use the manifold's names as written (e.g. `sell_through_rate`). Do not rename a
+  measure to something more familiar.
+
+## The manifold you are querying
+
+{MANIFOLD_CONTEXT}