datacharter 0.1.0__py3-none-any.whl

datacharter/__init__.py

@@ -0,0 +1,3 @@
+"""DataCharter — contract-governed local data exploration."""
+
+__version__ = "0.1.0"

datacharter/agent/__init__.py

@@ -0,0 +1,5 @@
+"""Natural-language agent: 5-tool loop over any OpenAI-compatible endpoint."""
+
+from datacharter.agent.loop import Agent, AgentConfig, AgentEvent
+
+__all__ = ["Agent", "AgentConfig", "AgentEvent"]

datacharter/agent/llm.py

@@ -0,0 +1,119 @@
+"""Minimal OpenAI-compatible chat client (httpx, no vendor SDK — DESIGN D2)."""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+__all__ = ["LLMClient", "LLMError", "ToolCall", "Delta"]
+
+
+class LLMError(Exception):
+    """LLM endpoint failure (connection, auth, or protocol)."""
+
+
+@dataclass
+class ToolCall:
+    id: str
+    name: str
+    arguments: str  # raw JSON string as emitted by the model
+
+
+@dataclass
+class Delta:
+    """One streamed step: text fragment and/or the finished tool calls."""
+
+    text: str = ""
+    tool_calls: list[ToolCall] | None = None
+
+
+class LLMClient:
+    """Talks to any /chat/completions endpoint (OpenAI, Ollama, vLLM, …)."""
+
+    def __init__(
+        self,
+        *,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        model: str | None = None,
+        timeout_s: float = 120.0,
+    ) -> None:
+        self.base_url = (
+            base_url or os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1")
+        ).rstrip("/")
+        self.api_key = api_key or os.environ.get("OPENAI_API_KEY", "")
+        self.model = model or os.environ.get("DATACHARTER_MODEL", "gpt-4o-mini")
+        self.timeout_s = timeout_s
+
+    async def stream(
+        self, messages: list[dict[str, Any]], tools: list[dict[str, Any]]
+    ) -> AsyncIterator[Delta]:
+        """Stream a completion, yielding text deltas and assembling tool calls."""
+        payload = {
+            "model": self.model,
+            "messages": messages,
+            "tools": tools,
+            "stream": True,
+        }
+        headers = {"content-type": "application/json"}
+        if self.api_key:
+            headers["authorization"] = f"Bearer {self.api_key}"
+
+        pending: dict[int, dict[str, str]] = {}
+        try:
+            async with (
+                httpx.AsyncClient(timeout=self.timeout_s) as client,
+                client.stream(
+                    "POST", f"{self.base_url}/chat/completions", json=payload, headers=headers
+                ) as resp,
+            ):
+                if resp.status_code >= 400:
+                    body = (await resp.aread()).decode("utf-8", "replace")[:500]
+                    raise LLMError(f"LLM endpoint returned {resp.status_code}: {body}")
+                async for line in resp.aiter_lines():
+                    delta = _parse_sse_line(line, pending)
+                    if delta is not None:
+                        yield delta
+        except httpx.HTTPError as exc:
+            raise LLMError(f"Could not reach LLM at {self.base_url}: {exc}") from None
+
+        if pending:
+            calls = [
+                ToolCall(id=c["id"], name=c["name"], arguments=c["arguments"])
+                for c in pending.values()
+                if c.get("name")
+            ]
+            if calls:
+                yield Delta(tool_calls=calls)
+
+
+def _parse_sse_line(line: str, pending: dict[int, dict[str, str]]) -> Delta | None:
+    if not line.startswith("data:"):
+        return None
+    data = line[len("data:") :].strip()
+    if not data or data == "[DONE]":
+        return None
+    try:
+        chunk = json.loads(data)
+    except json.JSONDecodeError:
+        return None
+    choices = chunk.get("choices") or [{}]
+    delta = choices[0].get("delta") or {}
+
+    text = delta.get("content") or ""
+    for tc in delta.get("tool_calls") or []:
+        idx = tc.get("index", 0)
+        slot = pending.setdefault(idx, {"id": "", "name": "", "arguments": ""})
+        if tc.get("id"):
+            slot["id"] = tc["id"]
+        fn = tc.get("function") or {}
+        if fn.get("name"):
+            slot["name"] = fn["name"]
+        if fn.get("arguments"):
+            slot["arguments"] += fn["arguments"]
+    return Delta(text=text) if text else None

datacharter/agent/loop.py

@@ -0,0 +1,103 @@
+"""The agent loop: stream text, execute tool calls, feed results back until done."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from datacharter.agent.llm import LLMClient, LLMError
+from datacharter.agent.tools import TOOL_SPECS, ToolBox
+
+__all__ = ["Agent", "AgentConfig", "AgentEvent"]
+
+MAX_TURNS = 8
+
+SYSTEM_PROMPT = """\
+You are DataCharter's data analyst. You answer questions about the user's data
+by exploring the catalog and running read-only SQL through your tools.
+
+Rules:
+- Discover before querying: use list_tables / describe_table to learn the
+  schema. Never guess column names.
+- Use fully-qualified relation names exactly as list_tables reports them.
+- Queries are read-only. Keep exploratory queries small with LIMIT.
+- Some columns are PII and come back masked as •••; never claim to reveal them.
+- When a result is best seen as a chart, end your answer with a fenced
+  ```vega-lite block containing a valid Vega-Lite spec whose data.values you
+  fill from your query results.
+- Be concise. Show the SQL you ran.
+"""
+
+
+@dataclass
+class AgentConfig:
+    llm: LLMClient = field(default_factory=LLMClient)
+    max_turns: int = MAX_TURNS
+
+
+@dataclass
+class AgentEvent:
+    kind: Literal["text", "tool_call", "tool_result", "error", "done"]
+    text: str = ""
+    tool: str = ""
+    detail: str = ""
+
+
+class Agent:
+    """One conversational turn = many LLM/tool rounds until a final answer."""
+
+    def __init__(self, tools: ToolBox, config: AgentConfig | None = None) -> None:
+        self._tools = tools
+        self._config = config or AgentConfig()
+
+    async def run(
+        self, question: str, history: list[dict[str, Any]] | None = None
+    ) -> AsyncIterator[AgentEvent]:
+        messages: list[dict[str, Any]] = [{"role": "system", "content": SYSTEM_PROMPT}]
+        messages.extend(history or [])
+        messages.append({"role": "user", "content": question})
+
+        for _turn in range(self._config.max_turns):
+            assistant_text = ""
+            tool_calls = None
+            try:
+                async for delta in self._config.llm.stream(messages, TOOL_SPECS):
+                    if delta.text:
+                        assistant_text += delta.text
+                        yield AgentEvent(kind="text", text=delta.text)
+                    if delta.tool_calls:
+                        tool_calls = delta.tool_calls
+            except LLMError as exc:
+                yield AgentEvent(kind="error", detail=str(exc))
+                return
+
+            if not tool_calls:
+                yield AgentEvent(kind="done")
+                return
+
+            messages.append(
+                {
+                    "role": "assistant",
+                    "content": assistant_text or None,
+                    "tool_calls": [
+                        {
+                            "id": c.id,
+                            "type": "function",
+                            "function": {"name": c.name, "arguments": c.arguments},
+                        }
+                        for c in tool_calls
+                    ],
+                }
+            )
+            for call in tool_calls:
+                yield AgentEvent(kind="tool_call", tool=call.name, detail=call.arguments)
+                result = await self._tools.run(call.name, call.arguments)
+                yield AgentEvent(kind="tool_result", tool=call.name, detail=result)
+                messages.append(
+                    {"role": "tool", "tool_call_id": call.id, "content": result}
+                )
+
+        yield AgentEvent(
+            kind="error", detail=f"Stopped after {self._config.max_turns} tool rounds."
+        )

datacharter/agent/tools.py

@@ -0,0 +1,146 @@
+"""Agent tools over the engine: catalog introspection + guarded querying with PII masking."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from datacharter.engine.session import Engine
+from datacharter.models import QueryResult, Source
+
+__all__ = ["ToolBox", "TOOL_SPECS"]
+
+TOOL_SPECS: list[dict[str, Any]] = [
+    {
+        "type": "function",
+        "function": {
+            "name": "list_sources",
+            "description": "List configured data sources with their types.",
+            "parameters": {"type": "object", "properties": {}},
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "list_tables",
+            "description": "List all queryable tables with their fully-qualified relation names.",
+            "parameters": {"type": "object", "properties": {}},
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "describe_table",
+            "description": "Show columns and types for one relation (e.g. 'crm.customers').",
+            "parameters": {
+                "type": "object",
+                "properties": {"relation": {"type": "string"}},
+                "required": ["relation"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "query",
+            "description": (
+                "Run a read-only SQL query and return rows. Use fully-qualified "
+                "relation names. Prefer LIMIT for exploration."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {"sql": {"type": "string"}},
+                "required": ["sql"],
+            },
+        },
+    },
+]
+
+MASKED = "•••"
+_MAX_TOOL_ROWS = 50
+
+
+class ToolBox:
+    """Executes tool calls against the engine; masks PII columns in returned data."""
+
+    def __init__(self, engine: Engine, sources: list[Source]) -> None:
+        self._engine = engine
+        # Column names flagged PII in any source's contract, matched case-insensitively.
+        self._pii: set[str] = set()
+        for src in sources:
+            for cols in src.pii.values():
+                self._pii.update(c.lower() for c in cols)
+
+    async def run(self, name: str, arguments: str) -> str:
+        try:
+            args = json.loads(arguments or "{}")
+        except json.JSONDecodeError:
+            return "Error: tool arguments were not valid JSON."
+        handler = {
+            "list_sources": self._list_sources,
+            "list_tables": self._list_tables,
+            "describe_table": self._describe_table,
+            "query": self._query,
+        }.get(name)
+        if handler is None:
+            return f"Error: unknown tool '{name}'."
+        try:
+            return await handler(args)
+        except Exception as exc:  # engine errors already scrubbed; surface to the model
+            return f"Error: {exc}"
+
+    async def _list_sources(self, _args: dict) -> str:
+        rows = [{"name": s.name, "type": s.type.value} for s in self._engine.sources]
+        return json.dumps(rows)
+
+    async def _list_tables(self, _args: dict) -> str:
+        result = await self._engine.query("SHOW ALL TABLES", timeout_s=30)
+        idx = {c: i for i, c in enumerate(result.columns)}
+        out = []
+        for row in result.rows:
+            db = row[idx["database"]]
+            if db in ("system", "temp"):
+                continue
+            table = row[idx["name"]]
+            relation = table if db == "memory" else f"{db}.{table}"
+            out.append({"relation": relation, "columns": list(row[idx["column_names"]])})
+        return json.dumps(out)
+
+    async def _describe_table(self, args: dict) -> str:
+        relation = str(args.get("relation", ""))
+        if not _is_safe_relation(relation):
+            return "Error: invalid relation name."
+        result = await self._engine.query(f"DESCRIBE {relation}", timeout_s=30)
+        return self._render(result)
+
+    async def _query(self, args: dict) -> str:
+        sql = str(args.get("sql", ""))
+        result = await self._engine.query(sql, row_limit=_MAX_TOOL_ROWS)
+        return self._render(result)
+
+    def _render(self, result: QueryResult) -> str:
+        mask_idx = {
+            i for i, col in enumerate(result.columns) if col.lower() in self._pii
+        }
+        rows = [
+            [MASKED if i in mask_idx else v for i, v in enumerate(row)]
+            for row in result.rows
+        ]
+        payload = {
+            "columns": result.columns,
+            "rows": rows,
+            "row_count": result.row_count,
+            "truncated": result.truncated,
+        }
+        if result.warnings:
+            payload["warnings"] = result.warnings
+        if mask_idx:
+            payload["masked_columns"] = [result.columns[i] for i in mask_idx]
+        return json.dumps(payload, default=str)
+
+
+def _is_safe_relation(relation: str) -> bool:
+    parts = relation.split(".")
+    return 1 <= len(parts) <= 3 and all(
+        p and all(ch.isalnum() or ch == "_" for ch in p) for p in parts
+    )

datacharter/cli.py

@@ -0,0 +1,249 @@
+"""Command-line entrypoint: init (workspace scaffolding), serve (M3)."""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import sys
+from pathlib import Path
+
+from datacharter import __version__
+
+CHARTER_TEMPLATE = """\
+# DataCharter workspace — https://github.com/datacharter/datacharter
+# Sources are data contracts: connection shape here, secrets NEVER here.
+# Credentials must be ${NAME} references resolved from your environment,
+# .env, or the OS keyring (`datacharter secrets set NAME`).
+version: 1
+
+sources: {}
+"""
+
+DEMO_CHARTER = """\
+# DataCharter demo workspace. Run: datacharter serve
+version: 1
+
+sources:
+  customers:
+    type: csv
+    path: demo/customers.csv
+    pii:
+      customers: [email]
+
+  orders:
+    type: parquet
+    path: demo/orders.parquet
+"""
+
+ENV_EXAMPLE = """\
+# Copy to .env and fill in real values. .env is gitignored.
+# EXAMPLE_DB_PASSWORD=change-me
+"""
+
+GITIGNORE_BLOCK = """\
+# DataCharter local state (never commit)
+.env
+.datacharter/
+"""
+
+
+def _cmd_init(args: argparse.Namespace) -> int:
+    ws = Path(args.directory).resolve()
+    ws.mkdir(parents=True, exist_ok=True)
+    charter = ws / "charter.yaml"
+    if charter.exists() and not args.force:
+        print(f"charter.yaml already exists in {ws} (use --force to overwrite).")
+        return 1
+
+    charter.write_text(DEMO_CHARTER if args.demo else CHARTER_TEMPLATE)
+    (ws / ".env.example").write_text(ENV_EXAMPLE)
+    (ws / "queries").mkdir(exist_ok=True)
+    _ensure_gitignore(ws)
+    if args.demo:
+        _write_demo_data(ws)
+    print(f"Workspace initialized in {ws}.")
+    if args.demo:
+        print("Demo data in demo/ — try: datacharter serve")
+    return 0
+
+
+def _ensure_gitignore(ws: Path) -> None:
+    gi = ws / ".gitignore"
+    existing = gi.read_text() if gi.exists() else ""
+    if ".datacharter/" not in existing:
+        joiner = "\n" if existing and not existing.endswith("\n") else ""
+        gi.write_text(existing + joiner + GITIGNORE_BLOCK)
+
+
+def _write_demo_data(ws: Path) -> None:
+    import duckdb
+
+    demo = ws / "demo"
+    demo.mkdir(exist_ok=True)
+    con = duckdb.connect()
+    con.execute(
+        "COPY (SELECT * FROM (VALUES "
+        "(1, 'ada@example.com', 'pro'), (2, 'grace@example.com', 'free'), "
+        "(3, 'edsger@example.com', 'pro')) t(id, email, tier)) TO "
+        f"'{demo / 'customers.csv'}' (FORMAT csv, HEADER)"
+    )
+    con.execute(
+        "COPY (SELECT (i % 3) + 1 AS customer_id, round(random() * 200, 2) AS total, "
+        "DATE '2026-01-01' + INTERVAL (i) DAY AS placed_on "
+        f"FROM range(90) t(i)) TO '{demo / 'orders.parquet'}' (FORMAT parquet)"
+    )
+    con.close()
+
+
+def _resolve_serve_workspace(directory: str) -> Path:
+    """Workspace for serve; ephemeral demo when no charter.yaml (D3, no surprise writes)."""
+    ws = Path(directory).resolve()
+    if (ws / "charter.yaml").exists():
+        return ws
+    import tempfile
+
+    demo_ws = Path(tempfile.mkdtemp(prefix="datacharter-demo-"))
+    (demo_ws / "charter.yaml").write_text(DEMO_CHARTER)
+    _write_demo_data(demo_ws)
+    print(f"No charter.yaml in {ws} — serving an ephemeral demo workspace ({demo_ws}).")
+    print("Run `datacharter init` here to start a real workspace.")
+    return demo_ws
+
+
+LOCAL_BASE_URL = "http://127.0.0.1:11434/v1"
+LOCAL_DEFAULT_MODEL = "qwen3:8b"
+
+
+def _local_llm(model: str | None):
+    """Point the agent at a local Ollama; verify it's reachable, hint if not."""
+    import httpx
+
+    from datacharter.agent.llm import LLMClient
+
+    chosen = model or LOCAL_DEFAULT_MODEL
+    try:
+        httpx.get("http://127.0.0.1:11434/api/tags", timeout=2.0).raise_for_status()
+    except Exception:
+        print("Ollama not reachable at 127.0.0.1:11434.")
+        print("Install from https://ollama.com, then: ollama pull " + chosen)
+        print("Serving without a local agent (the UI still works; chat will be disabled).")
+        return None
+    print(f"Local agent: Ollama model '{chosen}'. Pull it with: ollama pull {chosen}")
+    return LLMClient(base_url=LOCAL_BASE_URL, api_key="ollama", model=chosen)
+
+
+def _cmd_serve(args: argparse.Namespace) -> int:
+    import uvicorn
+
+    from datacharter.server import create_app
+
+    ws = _resolve_serve_workspace(args.directory)
+    llm = _local_llm(args.model) if args.local else None
+    app = create_app(ws, allow_spill=not args.no_spill, llm=llm, host=args.host)
+    print(f"DataCharter serving {ws} on http://{args.host}:{args.port}")
+    uvicorn.run(app, host=args.host, port=args.port, log_level="warning")
+    return 0
+
+
+_SECRETS_INDEX = "__datacharter_index__"  # reserved key tracking stored names for `list`
+
+
+def _secret_index(kr, service: str) -> list[str]:
+    try:
+        raw = kr.get_password(service, _SECRETS_INDEX)
+    except Exception:
+        return []
+    return [n for n in (raw or "").split(",") if n]
+
+
+def _cmd_secrets(args: argparse.Namespace) -> int:
+    import getpass
+
+    try:
+        import keyring
+    except ImportError:
+        print("The 'keyring' package is required for secret storage.", file=sys.stderr)
+        return 1
+    from datacharter.contracts.resolve import KEYRING_SERVICE
+
+    if args.action == "set":
+        value = args.value
+        if value is None:
+            value = getpass.getpass(f"Value for {args.name}: ")
+        if not value:
+            print("No value provided; nothing stored.", file=sys.stderr)
+            return 1
+        try:
+            keyring.set_password(KEYRING_SERVICE, args.name, value)
+            names = sorted(set(_secret_index(keyring, KEYRING_SERVICE)) | {args.name})
+            keyring.set_password(KEYRING_SERVICE, _SECRETS_INDEX, ",".join(names))
+        except Exception as exc:
+            print(f"Could not store secret (no keyring backend?): {exc}", file=sys.stderr)
+            return 1
+        print(f"Stored '{args.name}' in the OS keyring.")
+        return 0
+    if args.action == "rm":
+        with contextlib.suppress(Exception):
+            keyring.delete_password(KEYRING_SERVICE, args.name)
+        names = [n for n in _secret_index(keyring, KEYRING_SERVICE) if n != args.name]
+        with contextlib.suppress(Exception):
+            keyring.set_password(KEYRING_SERVICE, _SECRETS_INDEX, ",".join(names))
+        print(f"Removed '{args.name}'.")
+        return 0
+    names = _secret_index(keyring, KEYRING_SERVICE)
+    if names:
+        print("\n".join(names))
+    else:
+        print("No secrets stored via datacharter (env and .env are not listed here).")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="datacharter", description="Charter your data.")
+    parser.add_argument("--version", action="version", version=f"datacharter {__version__}")
+    sub = parser.add_subparsers(dest="command")
+
+    p_init = sub.add_parser(
+        "init", help="Scaffold a workspace (charter.yaml, queries/, .env.example)"
+    )
+    p_init.add_argument("directory", nargs="?", default=".")
+    p_init.add_argument("--demo", action="store_true", help="Include a generated demo dataset")
+    p_init.add_argument("--force", action="store_true", help="Overwrite an existing charter.yaml")
+    p_init.set_defaults(func=_cmd_init)
+
+    p_serve = sub.add_parser("serve", help="Start the local server")
+    p_serve.add_argument("directory", nargs="?", default=".")
+    p_serve.add_argument(
+        "--host", default="127.0.0.1", help="Bind address (default: localhost only)"
+    )
+    p_serve.add_argument("--port", type=int, default=8321)
+    p_serve.add_argument(
+        "--no-spill", action="store_true", help="Fail queries instead of spilling to disk"
+    )
+    p_serve.add_argument(
+        "--local", action="store_true", help="Use a local Ollama model for the agent"
+    )
+    p_serve.add_argument("--model", help="Model name (with --local; default qwen3:8b)")
+    p_serve.set_defaults(func=_cmd_serve)
+
+    p_secrets = sub.add_parser("secrets", help="Manage ${NAME} secrets in the OS keyring")
+    sec_sub = p_secrets.add_subparsers(dest="action")
+    p_set = sec_sub.add_parser("set", help="Store a secret (prompts if --value omitted)")
+    p_set.add_argument("name")
+    p_set.add_argument("--value", help="Value (omit to be prompted without echo)")
+    sec_sub.add_parser("rm", help="Remove a secret").add_argument("name")
+    sec_sub.add_parser("list", help="List secret names stored via datacharter")
+    p_secrets.set_defaults(func=_cmd_secrets)
+
+    args = parser.parse_args(argv)
+    if args.command == "secrets" and not getattr(args, "action", None):
+        p_secrets.print_help()
+        return 1
+    if not hasattr(args, "func"):
+        parser.print_help()
+        return 1
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

datacharter/contracts/__init__.py

@@ -0,0 +1,5 @@
+"""charter.yaml loading: parsing, secret resolution, validation, portability lint."""
+
+from datacharter.contracts.loader import Charter, CharterError, load_charter
+
+__all__ = ["Charter", "CharterError", "load_charter"]