magic-lab-observer-cli 0.1.1__tar.gz

magic_lab_observer_cli-0.1.1/.gitignore

@@ -0,0 +1,68 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Testing
+.coverage
+htmlcov/
+.pytest_cache/
+.mypy_cache/
+
+# Node.js
+node_modules/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Build
+dist/
+build/
+*.tsbuildinfo
+
+# Environment
+.env
+.env.local
+.env.*.local
+
+# Podman
+podman-compose.override.yml
+
+# Database
+*.db
+*.sqlite3
+.observer-state/
+
+# OS
+.DS_Store
+Thumbs.db
+

magic_lab_observer_cli-0.1.1/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: magic-lab-observer-cli
+Version: 0.1.1
+Summary: CLI for LLM Observatory
+Project-URL: Homepage, https://github.com/Magic-Lab-Studio/observer
+Project-URL: Repository, https://github.com/Magic-Lab-Studio/observer.git
+Project-URL: Issues, https://github.com/Magic-Lab-Studio/observer/issues
+License-Expression: Apache-2.0
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: rich>=13.0.0
+Description-Content-Type: text/markdown
+
+# LLM Observatory CLI
+
+Command-line interface for LLM Observatory.
+
+## Installation
+
+```bash
+pip install magic-lab-observer-cli
+```
+
+Before the distribution is published, install this directory from the checked-out
+repository with `python -m pip install -e ./cli`.
+
+## Usage
+
+```bash
+# Start the server
+llm-observatory serve
+
+# Run an evaluation
+llm-observatory evaluate --trace-id abc123 --evaluator llm_judge --criteria accuracy,safety
+
+# Export traces
+llm-observatory export --format json --output traces.json
+
+# Check server status
+llm-observatory status
+```
+
+## License
+
+Apache License 2.0

magic_lab_observer_cli-0.1.1/README.md

@@ -0,0 +1,32 @@
+# LLM Observatory CLI
+
+Command-line interface for LLM Observatory.
+
+## Installation
+
+```bash
+pip install magic-lab-observer-cli
+```
+
+Before the distribution is published, install this directory from the checked-out
+repository with `python -m pip install -e ./cli`.
+
+## Usage
+
+```bash
+# Start the server
+llm-observatory serve
+
+# Run an evaluation
+llm-observatory evaluate --trace-id abc123 --evaluator llm_judge --criteria accuracy,safety
+
+# Export traces
+llm-observatory export --format json --output traces.json
+
+# Check server status
+llm-observatory status
+```
+
+## License
+
+Apache License 2.0

magic_lab_observer_cli-0.1.1/cli/__init__.py

@@ -0,0 +1 @@
+"""LLM Observatory CLI - Command-line interface for LLM Observatory."""

magic_lab_observer_cli-0.1.1/cli/db.py

@@ -0,0 +1,188 @@
+"""Database helper for local SQLite access."""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+DEFAULT_DB_PATH = Path("observatory.db")
+
+# DDL to create tables if the database is fresh/empty
+_SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS traces (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    session_id TEXT,
+    start_time TIMESTAMP NOT NULL,
+    end_time TIMESTAMP,
+    status TEXT NOT NULL DEFAULT 'unset',
+    metadata JSON,
+    created_at TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS ix_traces_session_id ON traces (session_id);
+
+CREATE TABLE IF NOT EXISTS spans (
+    id TEXT PRIMARY KEY,
+    trace_id TEXT NOT NULL REFERENCES traces(id) ON DELETE CASCADE,
+    parent_span_id TEXT REFERENCES spans(id),
+    name TEXT NOT NULL,
+    span_type TEXT NOT NULL DEFAULT 'generic',
+    start_time TIMESTAMP NOT NULL,
+    end_time TIMESTAMP,
+    status TEXT NOT NULL DEFAULT 'unset',
+    input JSON,
+    output JSON,
+    tokens_input INTEGER,
+    tokens_output INTEGER,
+    cost_usd NUMERIC(10, 6),
+    metadata JSON,
+    attributes JSON
+);
+CREATE INDEX IF NOT EXISTS ix_spans_trace_id ON spans (trace_id);
+CREATE INDEX IF NOT EXISTS ix_spans_parent_span_id ON spans (parent_span_id);
+
+CREATE TABLE IF NOT EXISTS evaluations (
+    id TEXT PRIMARY KEY,
+    trace_id TEXT NOT NULL REFERENCES traces(id) ON DELETE CASCADE,
+    span_id TEXT REFERENCES spans(id) ON DELETE CASCADE,
+    evaluator_type TEXT NOT NULL,
+    score NUMERIC(5, 4),
+    criteria JSON,
+    result JSON,
+    created_at TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS ix_evaluations_trace_id ON evaluations (trace_id);
+CREATE INDEX IF NOT EXISTS ix_evaluations_span_id ON evaluations (span_id);
+"""
+
+
+def get_connection(db_path: Path | None = None) -> sqlite3.Connection:
+    """Get a SQLite connection and ensure schema exists."""
+    path = db_path or DEFAULT_DB_PATH
+    conn = sqlite3.connect(str(path))
+    conn.row_factory = sqlite3.Row
+    conn.executescript(_SCHEMA_SQL)
+    return conn
+
+
+def list_traces(
+    db_path: Path | None = None,
+    limit: int = 100,
+    offset: int = 0,
+    status: str | None = None,
+    name_search: str | None = None,
+) -> list[dict[str, Any]]:
+    """List traces from local SQLite database."""
+    conn = get_connection(db_path)
+    try:
+        query = "SELECT * FROM traces WHERE 1=1"
+        params: list[Any] = []
+
+        if status:
+            query += " AND status = ?"
+            params.append(status)
+
+        if name_search:
+            query += " AND name LIKE ?"
+            params.append(f"%{name_search}%")
+
+        query += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
+        params.extend([limit, offset])
+
+        cursor = conn.execute(query, params)
+        return [dict(row) for row in cursor.fetchall()]
+    finally:
+        conn.close()
+
+
+def get_trace(trace_id: str, db_path: Path | None = None) -> dict[str, Any] | None:
+    """Get a single trace by ID."""
+    conn = get_connection(db_path)
+    try:
+        cursor = conn.execute("SELECT * FROM traces WHERE id = ?", (trace_id,))
+        row = cursor.fetchone()
+        return dict(row) if row else None
+    finally:
+        conn.close()
+
+
+def get_trace_spans(trace_id: str, db_path: Path | None = None) -> list[dict[str, Any]]:
+    """Get all spans for a trace."""
+    conn = get_connection(db_path)
+    try:
+        cursor = conn.execute(
+            "SELECT * FROM spans WHERE trace_id = ? ORDER BY start_time",
+            (trace_id,),
+        )
+        return [dict(row) for row in cursor.fetchall()]
+    finally:
+        conn.close()
+
+
+def list_evaluations(
+    db_path: Path | None = None,
+    trace_id: str | None = None,
+    limit: int = 100,
+    offset: int = 0,
+) -> list[dict[str, Any]]:
+    """List evaluations from local SQLite database."""
+    conn = get_connection(db_path)
+    try:
+        query = "SELECT * FROM evaluations WHERE 1=1"
+        params: list[Any] = []
+
+        if trace_id:
+            query += " AND trace_id = ?"
+            params.append(trace_id)
+
+        query += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
+        params.extend([limit, offset])
+
+        cursor = conn.execute(query, params)
+        return [dict(row) for row in cursor.fetchall()]
+    finally:
+        conn.close()
+
+
+def get_stats(db_path: Path | None = None) -> dict[str, Any]:
+    """Get summary statistics from the database."""
+    conn = get_connection(db_path)
+    try:
+        stats: dict[str, Any] = {}
+
+        # Trace counts
+        cursor = conn.execute("SELECT COUNT(*) FROM traces")
+        stats["total_traces"] = cursor.fetchone()[0]
+
+        cursor = conn.execute("SELECT COUNT(*) FROM traces WHERE status = 'ok'")
+        stats["ok_traces"] = cursor.fetchone()[0]
+
+        cursor = conn.execute("SELECT COUNT(*) FROM traces WHERE status = 'error'")
+        stats["error_traces"] = cursor.fetchone()[0]
+
+        # Span counts
+        cursor = conn.execute("SELECT COUNT(*) FROM spans")
+        stats["total_spans"] = cursor.fetchone()[0]
+
+        # Token totals
+        cursor = conn.execute(
+            "SELECT SUM(tokens_input), SUM(tokens_output) FROM spans WHERE tokens_input IS NOT NULL"
+        )
+        row = cursor.fetchone()
+        stats["total_input_tokens"] = row[0] or 0
+        stats["total_output_tokens"] = row[1] or 0
+
+        # Cost total
+        cursor = conn.execute(
+            "SELECT SUM(cost_usd) FROM spans WHERE cost_usd IS NOT NULL"
+        )
+        stats["total_cost_usd"] = cursor.fetchone()[0] or 0.0
+
+        # Evaluation counts
+        cursor = conn.execute("SELECT COUNT(*) FROM evaluations")
+        stats["total_evaluations"] = cursor.fetchone()[0]
+
+        return stats
+    finally:
+        conn.close()

magic_lab_observer_cli-0.1.1/cli/main.py

@@ -0,0 +1,414 @@
+"""Main CLI application for LLM Observatory."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+@click.group()
+@click.version_option(version="0.1.1", prog_name="llm-observatory")
+@click.option("--api-key", envvar="OBSERVATORY_API_KEY", default=None, help="API key for backend authentication")
+@click.pass_context
+def cli(ctx: click.Context, api_key: str | None) -> None:
+    """LLM Observatory CLI - Open-source LLM observability."""
+    ctx.ensure_object(dict)
+    ctx.obj["api_key"] = api_key
+
+
+# ── Local Database Commands ──────────────────────────────────────────
+
+
+@cli.command()
+@click.option("--db", type=click.Path(exists=True), default="observatory.db", help="SQLite database path")
+@click.option("--limit", default=20, help="Number of traces to show")
+@click.option("--status", type=click.Choice(["ok", "error", "unset"]), help="Filter by status")
+@click.option("--search", help="Search trace names")
+def traces(db: str, limit: int, status: str | None, search: str | None) -> None:
+    """List traces from local SQLite database."""
+    from cli.db import list_traces
+
+    rows = list_traces(Path(db), limit=limit, status=status, name_search=search)
+
+    if not rows:
+        console.print("[yellow]No traces found[/yellow]")
+        return
+
+    table = Table(title=f"Traces ({len(rows)} shown)")
+    table.add_column("ID", style="cyan", no_wrap=True)
+    table.add_column("Name", style="green")
+    table.add_column("Status", style="yellow")
+    table.add_column("Spans", justify="right")
+    table.add_column("Created", style="blue")
+
+    for row in rows:
+        table.add_row(
+            str(row["id"])[:8],
+            str(row["name"]),
+            str(row["status"]),
+            "",
+            str(row.get("created_at", "")),
+        )
+
+    console.print(table)
+
+
+@cli.command()
+@click.argument("trace_id")
+@click.option("--db", type=click.Path(exists=True), default="observatory.db", help="SQLite database path")
+def inspect(trace_id: str, db: str) -> None:
+    """Inspect a trace and its spans."""
+    from cli.db import get_trace, get_trace_spans
+
+    trace_data = get_trace(trace_id, Path(db))
+    if not trace_data:
+        console.print(f"[red]Trace {trace_id} not found[/red]")
+        return
+
+    # Print trace info
+    console.print(f"\n[bold cyan]Trace: {trace_data['name']}[/bold cyan]")
+    console.print(f"  ID:       {trace_data['id']}")
+    console.print(f"  Status:   {trace_data['status']}")
+    console.print(f"  Created:  {trace_data.get('created_at', 'N/A')}")
+
+    # Print spans
+    spans = get_trace_spans(trace_id, Path(db))
+    if spans:
+        console.print(f"\n[bold]Spans ({len(spans)}):[/bold]")
+        table = Table()
+        table.add_column("ID", style="cyan", no_wrap=True)
+        table.add_column("Name", style="green")
+        table.add_column("Type", style="yellow")
+        table.add_column("Status")
+        table.add_column("Tokens In", justify="right")
+        table.add_column("Tokens Out", justify="right")
+        table.add_column("Cost", justify="right")
+
+        for span in spans:
+            table.add_row(
+                str(span["id"])[:8],
+                str(span["name"]),
+                str(span["span_type"]),
+                str(span["status"]),
+                str(span.get("tokens_input") or ""),
+                str(span.get("tokens_output") or ""),
+                f"${span['cost_usd']:.6f}" if span.get("cost_usd") else "",
+            )
+
+        console.print(table)
+    else:
+        console.print("[yellow]No spans found[/yellow]")
+
+
+@cli.command(name="import")
+@click.argument("file", type=click.Path(exists=True))
+@click.option("--db", type=click.Path(), default="observatory.db", help="SQLite database path")
+@click.option("--dry-run", is_flag=True, help="Preview without writing")
+def import_data(file: str, db: str, dry_run: bool) -> None:
+    """Import traces from a JSON export file."""
+    import json
+
+    from cli.db import get_connection
+
+    raw = json.loads(Path(file).read_text())
+    traces = raw.get("traces", [])
+
+    if not traces:
+        console.print("[yellow]No traces in file[/yellow]")
+        return
+
+    console.print(f"[blue]Importing {len(traces)} traces from {file}...[/blue]")
+
+    if dry_run:
+        for t in traces:
+            n = len(t.get("spans", []))
+            console.print(f"  {t.get('name', '?')} — {n} spans")
+        console.print("[yellow]Dry run, no changes written[/yellow]")
+        return
+
+    conn = get_connection(Path(db))
+    try:
+        imported = 0
+        for trace in traces:
+            conn.execute(
+                "INSERT OR IGNORE INTO traces (id, name, session_id, status, metadata, start_time, end_time, created_at) VALUES (?,?,?,?,?,?,?,?)",
+                (
+                    trace["id"],
+                    trace["name"],
+                    trace.get("session_id"),
+                    trace.get("status", "unset"),
+                    json.dumps(trace.get("metadata")),
+                    trace.get("start_time"),
+                    trace.get("end_time"),
+                    trace.get("created_at"),
+                ),
+            )
+            for span in trace.get("spans", []):
+                conn.execute(
+                    "INSERT OR IGNORE INTO spans (id, trace_id, parent_span_id, name, span_type, start_time, end_time, status, input, output, tokens_input, tokens_output, cost_usd, metadata, attributes) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+                    (
+                        span["id"],
+                        span.get("trace_id", trace["id"]),
+                        span.get("parent_span_id"),
+                        span["name"],
+                        span.get("span_type", "generic"),
+                        span.get("start_time"),
+                        span.get("end_time"),
+                        span.get("status", "unset"),
+                        json.dumps(span.get("input")) if span.get("input") else None,
+                        json.dumps(span.get("output")) if span.get("output") else None,
+                        span.get("tokens_input"),
+                        span.get("tokens_output"),
+                        span.get("cost_usd"),
+                        json.dumps(span.get("metadata")) if span.get("metadata") else None,
+                        json.dumps(span.get("attributes")) if span.get("attributes") else None,
+                    ),
+                )
+            imported += 1
+        conn.commit()
+        console.print(f"[green]Imported {imported} traces ({sum(len(t.get('spans', [])) for t in traces)} spans)[/green]")
+    finally:
+        conn.close()
+
+
+@cli.command()
+@click.option("--db", type=click.Path(exists=True), default="observatory.db", help="SQLite database path")
+def stats(db: str) -> None:
+    """Show summary statistics."""
+    from cli.db import get_stats
+
+    data = get_stats(Path(db))
+
+    console.print("\n[bold cyan]LLM Observatory Statistics[/bold cyan]\n")
+    console.print(f"  Total Traces:      {data['total_traces']}")
+    console.print(f"    OK:              {data['ok_traces']}")
+    console.print(f"    Error:           {data['error_traces']}")
+    console.print(f"  Total Spans:       {data['total_spans']}")
+    console.print(f"  Input Tokens:      {data['total_input_tokens']:,}")
+    console.print(f"  Output Tokens:     {data['total_output_tokens']:,}")
+    console.print(f"  Total Cost:        ${data['total_cost_usd']:.6f}")
+    console.print(f"  Evaluations:       {data['total_evaluations']}")
+    console.print()
+
+
+@cli.command()
+@click.argument("trace_id")
+@click.option("--db", type=click.Path(exists=True), default="observatory.db", help="SQLite database path")
+@click.option("--format", "output_format", type=click.Choice(["json", "table"]), default="table")
+def spans(trace_id: str, db: str, output_format: str) -> None:
+    """List spans for a trace."""
+    from cli.db import get_trace_spans
+
+    rows = get_trace_spans(trace_id, Path(db))
+
+    if not rows:
+        console.print("[yellow]No spans found[/yellow]")
+        return
+
+    if output_format == "json":
+        import json
+        console.print(json.dumps(rows, indent=2, default=str))
+    else:
+        table = Table(title=f"Spans for {trace_id[:8]}")
+        table.add_column("ID", style="cyan", no_wrap=True)
+        table.add_column("Name", style="green")
+        table.add_column("Type", style="yellow")
+        table.add_column("Status")
+        table.add_column("Duration (ms)", justify="right")
+        table.add_column("Tokens", justify="right")
+        table.add_column("Cost", justify="right")
+
+        for row in rows:
+            duration = ""
+            if row.get("start_time") and row.get("end_time"):
+                from datetime import datetime
+                try:
+                    start = datetime.fromisoformat(str(row["start_time"]))
+                    end = datetime.fromisoformat(str(row["end_time"]))
+                    duration = f"{(end - start).total_seconds() * 1000:.0f}"
+                except (ValueError, TypeError):
+                    pass
+
+            tokens = ""
+            if row.get("tokens_input") or row.get("tokens_output"):
+                tokens = f"{row.get('tokens_input', 0)}→{row.get('tokens_output', 0)}"
+
+            table.add_row(
+                str(row["id"])[:8],
+                str(row["name"]),
+                str(row["span_type"]),
+                str(row["status"]),
+                duration,
+                tokens,
+                f"${row['cost_usd']:.6f}" if row.get("cost_usd") else "",
+            )
+
+        console.print(table)
+
+
+# ── Server Commands ──────────────────────────────────────────────────
+
+
+@cli.command()
+@click.option("--host", default="0.0.0.0", help="Host to bind to")
+@click.option("--port", default=8000, help="Port to bind to")
+@click.option("--reload", is_flag=True, help="Enable auto-reload")
+@click.option("--db", default="sqlite+aiosqlite:///./observatory.db", help="Database URL")
+def serve(host: str, port: int, reload: bool, db: str) -> None:
+    """Start the observatory backend server."""
+    try:
+        import uvicorn
+    except ImportError:
+        console.print("[red]Error: uvicorn is not installed.[/red]")
+        console.print("Install it with: pip install 'magic-lab-observer-backend[sqlite]'")
+        return
+
+    os.environ["DATABASE_URL"] = db
+
+    console.print(f"[green]Starting LLM Observatory server on {host}:{port}[/green]")
+    console.print(f"[green]Database: {db}[/green]")
+
+    try:
+        from app.main import app as fastapi_app
+        uvicorn.run(fastapi_app, host=host, port=port, reload=reload)
+    except ImportError:
+        console.print("[red]Error: backend app module not found.[/red]")
+        console.print("Make sure the backend is installed: pip install -e ../backend")
+    except (RuntimeError, OSError) as e:
+        console.print(f"[red]Server error: {e}[/red]")
+
+
+@cli.command()
+@click.option("--trace-id", required=True, help="Trace ID to evaluate")
+@click.option(
+    "--evaluator",
+    type=click.Choice(["llm_judge", "rule_based", "human"]),
+    required=True,
+    help="Evaluator type",
+)
+@click.option("--criteria", help="Comma-separated criteria for evaluation")
+@click.option("--endpoint", default="http://localhost:8000", help="API endpoint")
+@click.pass_context
+def evaluate(
+    ctx: click.Context,
+    trace_id: str,
+    evaluator: str,
+    criteria: str | None,
+    endpoint: str,
+) -> None:
+    """Run an evaluation on a trace."""
+    import httpx
+
+    console.print(f"[blue]Evaluating trace {trace_id} with {evaluator}...[/blue]")
+
+    criteria_list = criteria.split(",") if criteria else []
+
+    headers: dict[str, str] = {}
+    api_key = ctx.obj.get("api_key")
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    try:
+        response = httpx.post(
+            f"{endpoint}/v1/evaluations/run",
+            json={
+                "trace_id": trace_id,
+                "evaluator_type": evaluator,
+                "criteria": [{"name": c.strip(), "description": c.strip()} for c in criteria_list] if criteria_list else [],
+            },
+            headers=headers,
+        )
+        response.raise_for_status()
+
+        result = response.json()
+        console.print("[green]Evaluation completed successfully![/green]")
+        console.print(f"Score: {result.get('score', 'N/A')}")
+        console.print(f"Passed: {result.get('passed', 'N/A')}")
+        if result.get("criteria"):
+            console.print("Criteria scores:")
+            for k, v in result["criteria"].items():
+                console.print(f"  {k}: {v}")
+    except httpx.HTTPError as e:
+        console.print(f"[red]Error: {e}[/red]")
+
+
+@cli.command()
+@click.option("--db", type=click.Path(exists=True), default="observatory.db", help="SQLite database path")
+@click.option("--format", "output_format", type=click.Choice(["json"]), default="json", help="Export format")
+@click.option("--output", "-o", type=click.Path(), help="Output file (default: stdout)")
+@click.option("--status", type=click.Choice(["ok", "error", "unset"]), help="Filter by status")
+@click.option("--limit", default=1000, help="Max traces to export")
+def export(db: str, output_format: str, output: str | None, status: str | None, limit: int) -> None:
+    """Export traces and spans to a file."""
+    import json
+
+    from cli.db import get_trace_spans, list_traces
+
+    rows = list_traces(Path(db), limit=limit, status=status)
+
+    if not rows:
+        console.print("[yellow]No traces to export[/yellow]")
+        return
+
+    traces_data = []
+    for row in rows:
+        trace_id = str(row["id"])
+        spans = get_trace_spans(trace_id, Path(db))
+        traces_data.append({
+            "id": trace_id,
+            "name": row["name"],
+            "status": row["status"],
+            "session_id": row.get("session_id"),
+            "start_time": row.get("start_time"),
+            "end_time": row.get("end_time"),
+            "metadata": json.loads(row["metadata"]) if row.get("metadata") else None,
+            "created_at": row.get("created_at"),
+            "spans": spans,
+        })
+
+    export_data = {
+        "format": "llm-observatory-export",
+        "version": "0.1.1",
+        "trace_count": len(traces_data),
+        "traces": traces_data,
+    }
+
+    json_str = json.dumps(export_data, indent=2, default=str)
+
+    if output:
+        Path(output).write_text(json_str)
+        console.print(f"[green]Exported {len(traces_data)} traces to {output}[/green]")
+    else:
+        console.print(json_str)
+
+
+@cli.command()
+@click.option("--endpoint", default="http://localhost:8000", help="API endpoint")
+@click.pass_context
+def status(ctx: click.Context, endpoint: str) -> None:
+    """Check the status of the observatory server."""
+    import httpx
+
+    headers: dict[str, str] = {}
+    api_key = ctx.obj.get("api_key")
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    try:
+        response = httpx.get(f"{endpoint}/health", headers=headers)
+        response.raise_for_status()
+
+        console.print("[green]Server is healthy![/green]")
+        console.print(f"Endpoint: {endpoint}")
+    except httpx.HTTPError:
+        console.print("[red]Server is not responding[/red]")
+
+
+if __name__ == "__main__":
+    cli()

magic_lab_observer_cli-0.1.1/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "magic-lab-observer-cli"
+version = "0.1.1"
+description = "CLI for LLM Observatory"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+dependencies = [
+    "click>=8.1.0",
+    "rich>=13.0.0",
+    "httpx>=0.27.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/Magic-Lab-Studio/observer"
+Repository = "https://github.com/Magic-Lab-Studio/observer.git"
+Issues = "https://github.com/Magic-Lab-Studio/observer/issues"
+
+[project.scripts]
+llm-observatory = "cli.main:cli"
+
+[tool.hatch.build.targets.wheel]
+packages = ["cli"]

magic_lab_observer_cli-0.1.1/tests/test_cli.py

@@ -0,0 +1,526 @@
+"""Tests for LLM Observatory CLI commands."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from cli.db import (
+    get_connection,
+    get_stats,
+    get_trace,
+    get_trace_spans,
+    list_evaluations,
+    list_traces,
+)
+from cli.main import cli
+from click.testing import CliRunner
+
+# ── Helpers ──────────────────────────────────────────────────────────
+
+def _seed_db(db: Path, *, traces: int = 3, spans_per_trace: int = 2) -> dict:
+    """Insert test data and return summary."""
+    conn = get_connection(db)
+    trace_ids = []
+    for i in range(traces):
+        tid = f"trace-{i:04d}"
+        conn.execute(
+            "INSERT INTO traces (id, name, status, start_time, end_time, created_at, session_id) "
+            "VALUES (?, ?, 'ok', '2025-01-01T00:00:00', '2025-01-01T00:00:01', ?, ?)",
+            (tid, f"test-trace-{i}", f"2025-01-01T00:00:0{i}", f"session-{i}"),
+        )
+        trace_ids.append(tid)
+        for j in range(spans_per_trace):
+            span_id = f"span-{i:04d}-{j:02d}"
+            conn.execute(
+                "INSERT INTO spans (id, trace_id, name, span_type, start_time, end_time, "
+                "status, tokens_input, tokens_output, cost_usd) "
+                "VALUES (?, ?, ?, 'llm_call', '2025-01-01T00:00:00', '2025-01-01T00:00:01', 'ok', ?, ?, ?)",
+                (span_id, tid, f"span-{j}", 100 + j, 200 + j, 0.001 * (j + 1)),
+            )
+    # Add one error trace
+    conn.execute(
+        "INSERT INTO traces (id, name, status, start_time, end_time, created_at) "
+        "VALUES ('trace-err', 'error-trace', 'error', '2025-01-01T00:00:00', '2025-01-01T00:00:01', '2025-01-01T00:00:02')"
+    )
+    conn.execute(
+        "INSERT INTO spans (id, trace_id, name, span_type, start_time, end_time, status, tokens_input, tokens_output, cost_usd) "
+        "VALUES ('span-err', 'trace-err', 'fail-span', 'llm_call', '2025-01-01T00:00:00', '2025-01-01T00:00:01', 'error', 50, 10, 0.005)"
+    )
+    # Add evaluation
+    conn.execute(
+        "INSERT INTO evaluations (id, trace_id, evaluator_type, score, created_at) "
+        "VALUES ('eval-1', 'trace-0000', 'rule_based', 0.85, '2025-01-01T00:00:03')"
+    )
+    conn.commit()
+    conn.close()
+    return {"traces": traces + 1, "total_spans": traces * spans_per_trace + 1}
+
+
+@pytest.fixture()
+def db_file(tmp_path: Path) -> Path:
+    """Create a seeded database in a temp directory."""
+    db = tmp_path / "test.db"
+    _seed_db(db)
+    return db
+
+
+@pytest.fixture()
+def empty_db(tmp_path: Path) -> Path:
+    """Create an empty (but schema-initialized) database."""
+    db = tmp_path / "empty.db"
+    get_connection(db).close()
+    return db
+
+
+@pytest.fixture()
+def runner() -> CliRunner:
+    return CliRunner()
+
+
+# ── db.py unit tests ─────────────────────────────────────────────────
+
+class TestDBHelpers:
+    def test_get_connection_creates_schema(self, tmp_path: Path) -> None:
+        db = tmp_path / "new.db"
+        conn = get_connection(db)
+        cursor = conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        tables = {row[0] for row in cursor.fetchall()}
+        conn.close()
+        assert "traces" in tables
+        assert "spans" in tables
+        assert "evaluations" in tables
+
+    def test_get_connection_idempotent(self, tmp_path: Path) -> None:
+        db = tmp_path / "idempotent.db"
+        get_connection(db).close()
+        get_connection(db).close()
+
+    def test_list_traces_empty(self, empty_db: Path) -> None:
+        assert list_traces(empty_db) == []
+
+    def test_list_traces_default(self, db_file: Path) -> None:
+        rows = list_traces(db_file)
+        assert len(rows) == 4  # 3 ok + 1 error
+
+    def test_list_traces_limit(self, db_file: Path) -> None:
+        rows = list_traces(db_file, limit=2)
+        assert len(rows) == 2
+
+    def test_list_traces_status_filter(self, db_file: Path) -> None:
+        rows = list_traces(db_file, status="error")
+        assert len(rows) == 1
+        assert rows[0]["name"] == "error-trace"
+
+    def test_list_traces_name_search(self, db_file: Path) -> None:
+        rows = list_traces(db_file, name_search="test-trace")
+        assert len(rows) == 3
+
+    def test_list_traces_name_search_no_match(self, db_file: Path) -> None:
+        rows = list_traces(db_file, name_search="nonexistent")
+        assert rows == []
+
+    def test_get_trace_found(self, db_file: Path) -> None:
+        t = get_trace("trace-0000", db_file)
+        assert t is not None
+        assert t["name"] == "test-trace-0"
+        assert t["status"] == "ok"
+
+    def test_get_trace_not_found(self, db_file: Path) -> None:
+        assert get_trace("no-such-id", db_file) is None
+
+    def test_get_trace_spans(self, db_file: Path) -> None:
+        spans = get_trace_spans("trace-0000", db_file)
+        assert len(spans) == 2
+        assert spans[0]["trace_id"] == "trace-0000"
+
+    def test_get_trace_spans_empty(self, db_file: Path) -> None:
+        spans = get_trace_spans("trace-err", db_file)
+        assert len(spans) == 1
+
+    def test_list_evaluations(self, db_file: Path) -> None:
+        evals = list_evaluations(db_file)
+        assert len(evals) == 1
+        assert evals[0]["evaluator_type"] == "rule_based"
+
+    def test_list_evaluations_filter_by_trace(self, db_file: Path) -> None:
+        evals = list_evaluations(db_file, trace_id="trace-0000")
+        assert len(evals) == 1
+
+    def test_list_evaluations_filter_no_match(self, db_file: Path) -> None:
+        evals = list_evaluations(db_file, trace_id="trace-nope")
+        assert evals == []
+
+    def test_get_stats(self, db_file: Path) -> None:
+        stats = get_stats(db_file)
+        assert stats["total_traces"] == 4
+        assert stats["ok_traces"] == 3
+        assert stats["error_traces"] == 1
+        assert stats["total_spans"] == 7
+        assert stats["total_evaluations"] == 1
+        assert stats["total_input_tokens"] > 0
+        assert stats["total_output_tokens"] > 0
+        assert stats["total_cost_usd"] > 0
+
+    def test_get_stats_empty(self, empty_db: Path) -> None:
+        stats = get_stats(empty_db)
+        assert stats["total_traces"] == 0
+        assert stats["total_spans"] == 0
+        assert stats["total_input_tokens"] == 0
+        assert stats["total_cost_usd"] == 0.0
+
+
+# ── CLI command tests ────────────────────────────────────────────────
+
+class TestCLIHelp:
+    def test_version(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["--version"])
+        assert r.exit_code == 0
+        assert "0.1.1" in r.output
+
+    def test_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["--help"])
+        assert r.exit_code == 0
+        assert "LLM Observatory CLI" in r.output
+
+    def test_traces_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["traces", "--help"])
+        assert r.exit_code == 0
+        assert "--limit" in r.output
+
+    def test_inspect_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["inspect", "--help"])
+        assert r.exit_code == 0
+        assert "TRACE_ID" in r.output
+
+    def test_stats_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["stats", "--help"])
+        assert r.exit_code == 0
+
+    def test_spans_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["spans", "--help"])
+        assert r.exit_code == 0
+        assert "--format" in r.output
+
+    def test_export_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["export", "--help"])
+        assert r.exit_code == 0
+        assert "--output" in r.output
+
+    def test_serve_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["serve", "--help"])
+        assert r.exit_code == 0
+        assert "--reload" in r.output
+
+    def test_evaluate_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["evaluate", "--help"])
+        assert r.exit_code == 0
+        assert "--trace-id" in r.output
+
+    def test_status_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["status", "--help"])
+        assert r.exit_code == 0
+        assert "--endpoint" in r.output
+
+
+class TestTracesCommand:
+    def test_traces_shows_table(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["traces", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "test-trace-0" in r.output
+        assert "Traces (4 shown)" in r.output
+
+    def test_traces_empty(self, runner: CliRunner, empty_db: Path) -> None:
+        r = runner.invoke(cli, ["traces", "--db", str(empty_db)])
+        assert r.exit_code == 0
+        assert "No traces found" in r.output
+
+    def test_traces_limit(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["traces", "--db", str(db_file), "--limit", "1"])
+        assert r.exit_code == 0
+        assert "Traces (1 shown)" in r.output
+
+    def test_traces_filter_status(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["traces", "--db", str(db_file), "--status", "error"])
+        assert r.exit_code == 0
+        assert "error-trace" in r.output
+
+    def test_traces_search(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["traces", "--db", str(db_file), "--search", "error"])
+        assert r.exit_code == 0
+        assert "error-trace" in r.output
+        assert "Traces (1 shown)" in r.output
+
+    def test_traces_missing_db(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["traces", "--db", "/nonexistent/path.db"])
+        assert r.exit_code != 0
+
+
+class TestInspectCommand:
+    def test_inspect_found(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["inspect", "trace-0000", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "test-trace-0" in r.output
+        assert "Spans (2)" in r.output
+
+    def test_inspect_not_found(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["inspect", "no-such", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "not found" in r.output
+
+    def test_inspect_shows_status(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["inspect", "trace-err", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "error" in r.output
+
+
+class TestStatsCommand:
+    def test_stats_output(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["stats", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "Total Traces:" in r.output
+        assert "4" in r.output
+        assert "OK:" in r.output
+        assert "3" in r.output
+        assert "Error:" in r.output
+
+    def test_stats_empty(self, runner: CliRunner, empty_db: Path) -> None:
+        r = runner.invoke(cli, ["stats", "--db", str(empty_db)])
+        assert r.exit_code == 0
+        assert "Total Traces:" in r.output
+        assert "0" in r.output
+
+    def test_stats_shows_tokens(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["stats", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "Input Tokens:" in r.output
+        assert "Output Tokens:" in r.output
+        assert "Total Cost:" in r.output
+
+
+class TestSpansCommand:
+    def test_spans_table(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["spans", "trace-0000", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "span-0" in r.output
+        assert "span-1" in r.output
+
+    def test_spans_json(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["spans", "trace-0000", "--db", str(db_file), "--format", "json"])
+        assert r.exit_code == 0
+        data = json.loads(r.output)
+        assert len(data) == 2
+        assert data[0]["trace_id"] == "trace-0000"
+
+    def test_spans_empty(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["spans", "trace-0000", "--db", str(db_file)])
+        assert r.exit_code == 0
+
+    def test_spans_no_match(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["spans", "no-such-trace", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "No spans found" in r.output
+
+    def test_spans_shows_duration(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["spans", "trace-0000", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "Duration (ms)" in r.output
+
+    def test_spans_shows_tokens(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["spans", "trace-0000", "--db", str(db_file)])
+        assert r.exit_code == 0
+        assert "Tokens" in r.output
+
+
+class TestExportCommand:
+    def test_export_stdout(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["export", "--db", str(db_file)])
+        assert r.exit_code == 0
+        data = json.loads(r.output)
+        assert data["format"] == "llm-observatory-export"
+        assert data["version"] == "0.1.1"
+        assert data["trace_count"] == 4
+
+    def test_export_to_file(self, runner: CliRunner, db_file: Path, tmp_path: Path) -> None:
+        out = tmp_path / "export.json"
+        r = runner.invoke(cli, ["export", "--db", str(db_file), "-o", str(out)])
+        assert r.exit_code == 0
+        assert out.exists()
+        data = json.loads(out.read_text())
+        assert data["trace_count"] == 4
+
+    def test_export_empty(self, runner: CliRunner, empty_db: Path) -> None:
+        r = runner.invoke(cli, ["export", "--db", str(empty_db)])
+        assert r.exit_code == 0
+        assert "No traces to export" in r.output
+
+    def test_export_filter_status(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["export", "--db", str(db_file), "--status", "error"])
+        assert r.exit_code == 0
+        data = json.loads(r.output)
+        assert data["trace_count"] == 1
+        assert data["traces"][0]["status"] == "error"
+
+    def test_export_includes_spans(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["export", "--db", str(db_file)])
+        data = json.loads(r.output)
+        trace_with_spans = next(t for t in data["traces"] if t["id"] == "trace-0000")
+        assert len(trace_with_spans["spans"]) == 2
+
+    def test_export_limit(self, runner: CliRunner, db_file: Path) -> None:
+        r = runner.invoke(cli, ["export", "--db", str(db_file), "--limit", "2"])
+        data = json.loads(r.output)
+        assert data["trace_count"] == 2
+
+    def test_export_roundtrip(self, runner: CliRunner, db_file: Path, tmp_path: Path) -> None:
+        """Export then re-import preserves data structure."""
+        out = tmp_path / "roundtrip.json"
+        r = runner.invoke(cli, ["export", "--db", str(db_file), "-o", str(out)])
+        assert r.exit_code == 0
+        data = json.loads(out.read_text())
+        for trace in data["traces"]:
+            assert "id" in trace
+            assert "name" in trace
+            assert "spans" in trace
+            for span in trace["spans"]:
+                assert "trace_id" in span
+                assert "name" in span
+
+
+class TestEvaluateCommand:
+    def test_evaluate_help_shows_options(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["evaluate", "--help"])
+        assert r.exit_code == 0
+        assert "llm_judge" in r.output
+        assert "rule_based" in r.output
+        assert "human" in r.output
+
+    def test_evaluate_requires_trace_id(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["evaluate", "--evaluator", "rule_based"])
+        assert r.exit_code != 0
+
+    def test_evaluate_requires_evaluator(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["evaluate", "--trace-id", "t1"])
+        assert r.exit_code != 0
+
+
+class TestStatusCommand:
+    def test_status_server_down(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["status", "--endpoint", "http://127.0.0.1:19999"])
+        assert r.exit_code == 0
+        assert "not responding" in r.output
+
+
+class TestAPIKeyOption:
+    def test_api_key_from_option(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["--api-key", "test-key", "status", "--endpoint", "http://127.0.0.1:19999"])
+        assert r.exit_code == 0
+        assert "not responding" in r.output
+
+    def test_api_key_none_by_default(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["status", "--endpoint", "http://127.0.0.1:19999"])
+        assert r.exit_code == 0
+
+
+class TestImportCommand:
+    def test_import_help(self, runner: CliRunner) -> None:
+        r = runner.invoke(cli, ["import", "--help"])
+        assert r.exit_code == 0
+        assert "Dry run" in r.output or "dry-run" in r.output
+
+    def test_import_file_not_found(self, runner: CliRunner, tmp_path: Path) -> None:
+        r = runner.invoke(cli, ["import", str(tmp_path / "nonexistent.json")])
+        assert r.exit_code != 0
+
+    def test_import_dry_run(self, runner: CliRunner, tmp_path: Path) -> None:
+        export_file = tmp_path / "export.json"
+        export_data = {
+            "format": "llm-observatory-export",
+            "version": "0.1.0",
+            "trace_count": 2,
+            "traces": [
+                {
+                    "id": "imp-001",
+                    "name": "imported-trace-1",
+                    "status": "ok",
+                    "session_id": "s1",
+                    "start_time": "2025-06-01T00:00:00",
+                    "end_time": "2025-06-01T00:00:01",
+                    "created_at": "2025-06-01T00:00:00",
+                    "spans": [
+                        {"id": "sp-001", "name": "llm-call", "span_type": "llm",
+                         "start_time": "2025-06-01T00:00:00", "end_time": "2025-06-01T00:00:01",
+                         "status": "ok", "tokens_input": 100, "tokens_output": 50,
+                         "cost_usd": 0.001},
+                    ],
+                },
+                {
+                    "id": "imp-002",
+                    "name": "imported-trace-2",
+                    "status": "error",
+                    "spans": [],
+                },
+            ],
+        }
+        export_file.write_text(json.dumps(export_data))
+
+        r = runner.invoke(cli, ["import", str(export_file), "--dry-run"])
+        assert r.exit_code == 0
+        assert "imported-trace-1" in r.output
+        assert "Dry run" in r.output
+
+        # Nothing should be written
+        db = tmp_path / "test.db"
+        conn = get_connection(db)
+        cursor = conn.execute("SELECT COUNT(*) FROM traces")
+        assert cursor.fetchone()[0] == 0
+        conn.close()
+
+    def test_import_writes_to_db(self, runner: CliRunner, tmp_path: Path) -> None:
+        export_file = tmp_path / "export.json"
+        export_data = {
+            "format": "llm-observatory-export",
+            "version": "0.1.0",
+            "trace_count": 1,
+            "traces": [
+                {
+                    "id": "imp-100",
+                    "name": "imported-ok",
+                    "status": "ok",
+                    "session_id": "s1",
+                    "start_time": "2025-06-01T00:00:00",
+                    "end_time": "2025-06-01T00:00:01",
+                    "created_at": "2025-06-01T00:00:00",
+                    "spans": [
+                        {"id": "sp-100", "name": "call-1", "span_type": "llm",
+                         "start_time": "2025-06-01T00:00:00", "end_time": "2025-06-01T00:00:01",
+                         "status": "ok", "tokens_input": 200, "tokens_output": 100,
+                         "cost_usd": 0.005},
+                    ],
+                },
+            ],
+        }
+        export_file.write_text(json.dumps(export_data))
+
+        db = tmp_path / "import.db"
+        r = runner.invoke(cli, ["import", str(export_file), "--db", str(db)])
+        assert r.exit_code == 0
+        assert "Imported 1 traces" in r.output
+
+        # Verify data was written
+        conn = get_connection(db)
+        cursor = conn.execute("SELECT COUNT(*) FROM traces")
+        assert cursor.fetchone()[0] == 1
+        cursor = conn.execute("SELECT name FROM traces WHERE id='imp-100'")
+        assert cursor.fetchone()[0] == "imported-ok"
+        cursor = conn.execute("SELECT COUNT(*) FROM spans WHERE trace_id='imp-100'")
+        assert cursor.fetchone()[0] == 1
+        conn.close()
+
+    def test_import_empty_file(self, runner: CliRunner, tmp_path: Path) -> None:
+        export_file = tmp_path / "empty.json"
+        export_file.write_text(json.dumps({"format": "llm-observatory-export", "traces": []}))
+        r = runner.invoke(cli, ["import", str(export_file)])
+        assert r.exit_code == 0
+        assert "No traces" in r.output