spooling 0.1.8__tar.gz → 0.2.0__tar.gz

{spooling-0.1.8/spooling.egg-info → spooling-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spooling
-Version: 0.1.8
+Version: 0.2.0
 Summary: Local session tracker and semantic search for AI coding assistants
 Author: Parsed Analytics, Inc.
 License: MIT
@@ -11,8 +11,8 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: click>=8.1
-Requires-Dist: psycopg[binary]>=3.1
 Requires-Dist: sentence-transformers>=3.0
+Requires-Dist: numpy>=1.24
 Requires-Dist: fastapi>=0.111
 Requires-Dist: uvicorn[standard]>=0.30
 Requires-Dist: jinja2>=3.1
@@ -22,7 +22,7 @@ Requires-Dist: httpx>=0.27
 Requires-Dist: anthropic>=0.40
 Requires-Dist: strands-agents[ollama]>=1.35
 Requires-Dist: strands-agents-evals>=0.1
-Requires-Dist: mcp>=1.27
+Requires-Dist: mcp<2,>=1.27
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: pytest-mock>=3.14; extra == "dev"

{spooling-0.1.8 → spooling-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "spooling"
-version = "0.1.8"
+version = "0.2.0"
 description = "Local session tracker and semantic search for AI coding assistants"
 readme = "README.md"
 authors = [{ name = "Parsed Analytics, Inc." }]
@@ -13,8 +13,8 @@ classifiers = [
 ]
 dependencies = [
     "click>=8.1",
-    "psycopg[binary]>=3.1",
     "sentence-transformers>=3.0",
+    "numpy>=1.24",
     "fastapi>=0.111",
     "uvicorn[standard]>=0.30",
     "jinja2>=3.1",
@@ -24,7 +24,7 @@ dependencies = [
     "anthropic>=0.40",
     "strands-agents[ollama]>=1.35",
     "strands-agents-evals>=0.1",
-    "mcp>=1.27",
+    "mcp>=1.27,<2",
 ]
 
 [project.optional-dependencies]

{spooling-0.1.8 → spooling-0.2.0}/spooling/cli.py

@@ -26,18 +26,17 @@ def cli():
 def init():
     """Check database connection and show provider status."""
     from spooling.db import check_db
-    from spooling.config import DATABASE_URL
     from spooling.providers import get_all_providers
 
     console.print(Panel("[bold]Spooling[/bold] - Session Tracker", style="blue"))
 
     # Check DB
+    from spooling.config import DB_PATH
     if check_db():
-        console.print("[green]Database connected[/green]")
+        console.print(f"[green]Database ready[/green] [dim]({DB_PATH})[/dim]")
     else:
-        console.print("[red]Cannot connect to database.[/red]")
-        console.print(f"  URL: {DATABASE_URL}")
-        console.print("  Run: [bold]docker compose up -d[/bold]")
+        console.print(f"[red]Cannot open database at {DB_PATH}.[/red]")
+        console.print("  Check that the directory is writable.")
         return
 
     # Check all providers
@@ -215,7 +214,14 @@ def stats(week, days, cloud_mode):
         table.add_column("Cost", justify="right")
         for r in overview["recent_sessions"]:
             proj = _clean_project(r["project"] or "")
-            ts = r["started_at"].strftime("%m/%d %H:%M") if r["started_at"] else ""
+            raw_ts = r["started_at"]
+            if raw_ts and isinstance(raw_ts, str):
+                from datetime import datetime as _dt
+                try:
+                    raw_ts = _dt.fromisoformat(raw_ts)
+                except ValueError:
+                    raw_ts = None
+            ts = raw_ts.strftime("%m/%d %H:%M") if raw_ts else ""
             title = (r["title"] or "")[:50]
             table.add_row(
                 ts, proj, title,
@@ -684,27 +690,48 @@ def pricing_show(model):
 def tunnel(port, name):
     """Start a Cloudflare tunnel to expose a local MCP server.
 
-    Creates a quick tunnel (no account required) that exposes your local
-    MCP server to the internet. The tunnel URL can be used by any
-    MCP-compatible client.
+    Creates a quick tunnel (no account required) that exposes your local MCP
+    server to the internet. The tunnel URL can be used by any
+    MCP-compatible client on any device.
+
+    If a tunnel token has been generated (via [bold]spooling token generate[/bold]),
+    the printed config snippet includes the Bearer auth header automatically.
 
     Examples:
     \b
     spooling tunnel                    # Tunnel Spooling MCP (port 3004)
     spooling tunnel --port 8090        # Tunnel a custom MCP server
-    spooling tunnel -p 3000 -n my-api  # Named tunnel
     """
+    import json as _json
     from spooling.tunnel import start_tunnel
+    from spooling.tunnel_auth import load_token
+
+    tok = load_token()
+
+    if tok:
+        console.print()
+        console.print(f"[bold cyan]Tunnel token:[/bold cyan] [bold]{tok}[/bold]")
+        console.print("[dim]The MCP server will require this token on every request.[/dim]")
+        console.print()
+    else:
+        console.print()
+        console.print("[yellow]No tunnel token configured — the endpoint will be unauthenticated.[/yellow]")
+        console.print("[dim]Run [bold]spooling token generate[/bold] then restart [bold]spooling mcp[/bold] to secure it.[/dim]")
+        console.print()
 
     display_name = name or f"port {port}"
     console.print(f"[bold]Starting tunnel for {display_name}...[/bold]")
 
     url = start_tunnel(port=port, name=name)
     if url:
-        # Print MCP config snippet for easy copy-paste
+        # Build MCP config snippet — include auth header when a token is set.
+        mcp_server_config: dict = {"type": "http", "url": f"{url}/mcp"}
+        if tok:
+            mcp_server_config["headers"] = {"Authorization": f"Bearer {tok}"}
+        snippet = _json.dumps({"mcpServers": {"spooling": mcp_server_config}}, indent=2)
         console.print()
         console.print("[dim]Add this to your MCP client config:[/dim]")
-        console.print(f'[bold]{{"mcpServers": {{"custom": {{"type": "http", "url": "{url}/mcp"}}}}}}[/bold]')
+        console.print(f"[bold]{snippet}[/bold]")
         console.print()
 
 
@@ -826,5 +853,90 @@ cli.add_command(_cloud_group)
 cli.add_command(_push_cmd)
 
 
+# ---------------------------------------------------------------------------
+# Token management
+# ---------------------------------------------------------------------------
+
+@cli.group()
+def token():
+    """Manage the MCP tunnel authentication token.
+
+    The token secures the public tunnel endpoint so only authorised clients
+    can query your sessions.  Generate it once, then restart the MCP server
+    to enforce it.
+
+    \b
+    Quick-start:
+      spooling token generate   # create & save a token
+      spooling mcp              # MCP server now requires the token
+      spooling tunnel           # shows the config snippet with the token
+    """
+
+
+@token.command("generate")
+@click.option("--force", "-f", is_flag=True, help="Replace an existing token without prompting.")
+def token_generate(force):
+    """Generate and save a new tunnel token.
+
+    Replaces any existing token.  Restart the MCP server afterwards so it
+    picks up the new value.
+    """
+    from spooling.tunnel_auth import generate_token, load_token, save_token
+
+    existing = load_token()
+    if existing and not force:
+        console.print("[yellow]A token already exists.[/yellow]  Use [bold]--force[/bold] to replace it, or [bold]spooling token show[/bold] to view it.")
+        return
+
+    tok = generate_token()
+    save_token(tok)
+    console.print()
+    console.print("[green]Token generated and saved.[/green]")
+    console.print()
+    console.print(f"  [bold cyan]Token:[/bold cyan] [bold]{tok}[/bold]")
+    console.print()
+    console.print("[dim]Next steps:[/dim]")
+    console.print("  1. Restart [bold]spooling mcp[/bold] so it enforces the token.")
+    console.print("  2. Run [bold]spooling tunnel[/bold] to get a ready-to-paste MCP config snippet.")
+    console.print()
+
+
+@token.command("show")
+def token_show():
+    """Print the current tunnel token."""
+    from spooling.tunnel_auth import load_token, TOKEN_ENV_VAR, _TOKEN_FILE
+
+    tok = load_token()
+    if not tok:
+        console.print("[yellow]No token configured.[/yellow]  Run [bold]spooling token generate[/bold] to create one.")
+        return
+
+    source = f"env var {TOKEN_ENV_VAR}" if __import__("os").environ.get(TOKEN_ENV_VAR) else str(_TOKEN_FILE)
+    console.print()
+    console.print(f"  [bold cyan]Token:[/bold cyan] [bold]{tok}[/bold]")
+    console.print(f"  [dim]Source: {source}[/dim]")
+    console.print()
+
+
+@token.command("rotate")
+def token_rotate():
+    """Generate a new token, replacing the current one.
+
+    Equivalent to [bold]spooling token generate --force[/bold].
+    Restart the MCP server afterwards.
+    """
+    from spooling.tunnel_auth import generate_token, save_token
+
+    tok = generate_token()
+    save_token(tok)
+    console.print()
+    console.print("[green]Token rotated.[/green]")
+    console.print()
+    console.print(f"  [bold cyan]New token:[/bold cyan] [bold]{tok}[/bold]")
+    console.print()
+    console.print("[dim]Restart [bold]spooling mcp[/bold] to enforce the new token.[/dim]")
+    console.print()
+
+
 if __name__ == "__main__":
     cli()

{spooling-0.1.8 → spooling-0.2.0}/spooling/config.py

@@ -1,32 +1,39 @@
 """Configuration for Spooling."""
 
 import os
+import sys
 from pathlib import Path
 
 # Legacy session data directory (JSONL-format sessions)
 SESSIONS_DIR = Path.home() / ".sessions"
 SESSIONS_PROJECTS_DIR = SESSIONS_DIR / "projects"
 
-# Snowflake Cortex Code data directory. Sessions live in
-# ~/.snowflake/cortex/conversations/<uuid>.history.jsonl with a sidecar
-# <uuid>.json carrying title, working_directory, git info, and timestamps.
+# Snowflake Cortex Code data directory.
 CORTEX_DIR = Path.home() / ".snowflake" / "cortex"
 CORTEX_CONVERSATIONS_DIR = CORTEX_DIR / "conversations"
 
 # opencode (sst/opencode) data directory. Single SQLite DB at
-# ~/.local/share/opencode/opencode.db with session/message/part tables
-# (Drizzle-managed). Parts carry the Vercel AI SDK UIMessage payload.
+# ~/.local/share/opencode/opencode.db
 OPENCODE_DIR = Path.home() / ".local" / "share" / "opencode"
 OPENCODE_DB = OPENCODE_DIR / "opencode.db"
 
-# Database
-DB_HOST = os.getenv("SPOOLING_DB_HOST", "localhost")
-DB_PORT = int(os.getenv("SPOOLING_DB_PORT", "5432"))
-DB_NAME = os.getenv("SPOOLING_DB_NAME", "spooling")
-DB_USER = os.getenv("SPOOLING_DB_USER", "spooling")
-DB_PASSWORD = os.getenv("SPOOLING_DB_PASSWORD", "spooling")
-
-DATABASE_URL = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
+# ---------------------------------------------------------------------------
+# SQLite database path
+# ---------------------------------------------------------------------------
+# XDG-style on Linux/macOS: ~/.local/share/spooling/spooling.db
+# Windows:                   %APPDATA%\spooling\spooling.db
+# Override with SPOOLING_DB env var.
+
+def _default_db_path() -> Path:
+    if env := os.getenv("SPOOLING_DB"):
+        return Path(env)
+    if sys.platform == "win32":
+        base = Path(os.environ.get("APPDATA", Path.home()))
+    else:
+        base = Path.home() / ".local" / "share"
+    return base / "spooling" / "spooling.db"
+
+DB_PATH: Path = _default_db_path()
 
 # Embeddings
 EMBEDDING_MODEL = os.getenv("SPOOLING_EMBEDDING_MODEL", "all-MiniLM-L6-v2")

spooling-0.2.0/spooling/db.py

@@ -0,0 +1,164 @@
+"""SQLite database connection with PostgreSQL compatibility shim.
+
+Replaces the old psycopg/PostgreSQL backend so `pip install spooling`
+works without Docker or any system dependencies.
+
+The shim translates the subset of PostgreSQL SQL idioms used in this
+codebase to their SQLite equivalents at query time:
+
+  %s           → ?            (parameter placeholder)
+  now()        → datetime('now')
+  ::typename   → (stripped)   (%s::vector, %s::jsonb, etc.)
+  ILIKE        → LIKE         (SQLite LIKE is case-insensitive for ASCII)
+"""
+
+import re
+import sqlite3
+from pathlib import Path
+
+from spooling.config import DB_PATH
+from spooling.schema import SCHEMA_SQL
+
+
+# ---------------------------------------------------------------------------
+# SQL compatibility translation
+# ---------------------------------------------------------------------------
+
+_PLACEHOLDER_RE = re.compile(r"%s")
+_CAST_RE = re.compile(r"::\w+")
+_ILIKE_RE = re.compile(r"\bILIKE\b", re.IGNORECASE)
+
+
+def _adapt_sql(sql: str) -> str:
+    """Translate PostgreSQL SQL idioms → SQLite equivalents."""
+    sql = _PLACEHOLDER_RE.sub("?", sql)
+    sql = sql.replace("now()", "datetime('now')")
+    sql = _CAST_RE.sub("", sql)
+    sql = _ILIKE_RE.sub("LIKE", sql)
+    return sql
+
+
+# ---------------------------------------------------------------------------
+# Row and cursor wrappers
+# ---------------------------------------------------------------------------
+
+class _Row(dict):
+    """dict subclass that also supports attribute access and .get()."""
+
+    def __getattr__(self, key: str):
+        try:
+            return self[key]
+        except KeyError:
+            raise AttributeError(key) from None
+
+
+def _row_factory(cursor: sqlite3.Cursor, row: tuple) -> _Row:
+    return _Row(zip((col[0] for col in cursor.description), row))
+
+
+class _Cursor:
+    """Thin wrapper around sqlite3.Cursor with psycopg-style execute()."""
+
+    __slots__ = ("_cur",)
+
+    def __init__(self, cur: sqlite3.Cursor) -> None:
+        self._cur = cur
+
+    def execute(self, sql: str, params=()) -> "_Cursor":
+        self._cur.execute(_adapt_sql(sql), params)
+        return self
+
+    def executemany(self, sql: str, seq) -> "_Cursor":
+        self._cur.executemany(_adapt_sql(sql), seq)
+        return self
+
+    def fetchone(self) -> _Row | None:
+        return self._cur.fetchone()
+
+    def fetchall(self) -> list[_Row]:
+        return self._cur.fetchall()
+
+    @property
+    def lastrowid(self) -> int | None:
+        return self._cur.lastrowid
+
+    @property
+    def rowcount(self) -> int:
+        return self._cur.rowcount
+
+
+class SpoolingConnection:
+    """Wraps sqlite3.Connection with a psycopg-compatible interface."""
+
+    __slots__ = ("_conn",)
+
+    def __init__(self, conn: sqlite3.Connection) -> None:
+        conn.row_factory = _row_factory
+        self._conn = conn
+
+    # ---- core methods ----
+
+    def execute(self, sql: str, params=()) -> _Cursor:
+        return _Cursor(self._conn.execute(_adapt_sql(sql), params))
+
+    def executemany(self, sql: str, seq) -> _Cursor:
+        return _Cursor(self._conn.executemany(_adapt_sql(sql), seq))
+
+    def cursor(self) -> _Cursor:
+        return _Cursor(self._conn.cursor())
+
+    def commit(self) -> None:
+        self._conn.commit()
+
+    def rollback(self) -> None:
+        self._conn.rollback()
+
+    def close(self) -> None:
+        self._conn.close()
+
+    # ---- context manager ----
+
+    def __enter__(self) -> "SpoolingConnection":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        if exc_type is None:
+            self._conn.commit()
+        else:
+            self._conn.rollback()
+        self._conn.close()
+
+
+# ---------------------------------------------------------------------------
+# Schema bootstrap
+# ---------------------------------------------------------------------------
+
+def _ensure_schema(conn: sqlite3.Connection) -> None:
+    """Create all tables/indexes if they don't exist yet."""
+    conn.executescript(SCHEMA_SQL)
+    conn.commit()
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def get_connection() -> SpoolingConnection:
+    """Open (and auto-create if needed) the local SQLite database."""
+    DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+    raw = sqlite3.connect(str(DB_PATH), check_same_thread=False)
+    raw.execute("PRAGMA journal_mode=WAL")
+    raw.execute("PRAGMA foreign_keys=ON")
+    _ensure_schema(raw)
+    return SpoolingConnection(raw)
+
+
+def check_db() -> bool:
+    """Return True if the database is reachable (always True for SQLite)."""
+    try:
+        conn = get_connection()
+        conn.execute("SELECT 1")
+        conn.close()
+        return True
+    except Exception:
+        return False

{spooling-0.1.8 → spooling-0.2.0}/spooling/ingest.py

@@ -1,6 +1,7 @@
-"""Ingestion pipeline - parse AI coding sessions from multiple providers and store in pgvector."""
+"""Ingestion pipeline - parse AI coding sessions from multiple providers and store in SQLite."""
 
 import json
+import struct
 from pathlib import Path
 
 from rich.console import Console
@@ -236,27 +237,43 @@ def _store_trace(conn, trace: Trace):
     # still benefit, but sessions with zero llm_call cost keep their
     # existing chars/4 estimate untouched.
     if trace.session_id:
+        # SQLite-compatible correlated UPDATE: compute the span cost in a
+        # subquery and write it back only when it is positive.
         conn.execute(
-            """UPDATE sessions ss
-               SET estimated_cost_usd = sub.span_cost
-               FROM (
-                 SELECT t.session_id, SUM(s.cost_usd)::numeric(10, 4) AS span_cost
-                 FROM spans s JOIN traces t ON s.trace_id = t.id
-                 WHERE t.session_id = %s
+            """UPDATE sessions
+               SET estimated_cost_usd = (
+                 SELECT SUM(s.cost_usd)
+                 FROM spans s
+                 JOIN traces t ON s.trace_id = t.id
+                 WHERE t.session_id = sessions.id
                    AND s.kind = 'llm_call'
                    AND s.cost_usd IS NOT NULL
                    AND s.cost_usd > 0
                    AND s.model IS NOT NULL
                    AND s.model <> '<synthetic>'
-                 GROUP BY t.session_id
-               ) sub
-               WHERE ss.id = sub.session_id AND sub.span_cost > 0""",
+               )
+               WHERE id = %s
+               AND (
+                 SELECT COALESCE(SUM(s.cost_usd), 0)
+                 FROM spans s
+                 JOIN traces t ON s.trace_id = t.id
+                 WHERE t.session_id = sessions.id
+                   AND s.kind = 'llm_call'
+                   AND s.cost_usd > 0
+                   AND s.model IS NOT NULL
+                   AND s.model <> '<synthetic>'
+               ) > 0""",
             (trace.session_id,),
         )
 
 
+def _serialize_embedding(vec: list[float]) -> bytes:
+    """Pack a float list to a compact float32 BLOB for SQLite storage."""
+    return struct.pack(f"{len(vec)}f", *vec)
+
+
 def _embed_session(conn, session: ParsedSession):
-    """Chunk and embed session messages into pgvector."""
+    """Chunk and embed session messages, storing vectors as BLOBs."""
     # Delete existing chunks for this session (re-embed on update)
     conn.execute("DELETE FROM chunks WHERE session_id = %s", (session.session_id,))
 
@@ -286,11 +303,12 @@ def _embed_session(conn, session: ParsedSession):
     for chunk, vec, meta in zip(all_chunks, vectors, chunk_meta):
         conn.execute(
             """INSERT INTO chunks (session_id, message_id, content, role, project, timestamp, embedding)
-               VALUES (%s, %s, %s, %s, %s, %s, %s::vector)""",
+               VALUES (%s, %s, %s, %s, %s, %s, %s)""",
             (
                 meta["session_id"], meta["message_id"], chunk,
-                meta["role"], meta["project"], meta["timestamp"],
-                str(vec),
+                meta["role"], meta["project"],
+                meta["timestamp"].isoformat() if hasattr(meta["timestamp"], "isoformat") else meta["timestamp"],
+                _serialize_embedding(vec),
             ),
         )
 

{spooling-0.1.8 → spooling-0.2.0}/spooling/mcp_server.py

@@ -29,6 +29,10 @@ from typing import Any, Optional
 
 import httpx
 from mcp.server.fastmcp import FastMCP
+from mcp.server.transport_security import TransportSecuritySettings
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
 
 from spooling.db import get_connection
 
@@ -127,6 +131,9 @@ mcp = FastMCP(
     host=MCP_HOST,
     port=MCP_PORT,
     stateless_http=True,
+    transport_security=TransportSecuritySettings(
+        enable_dns_rebinding_protection=False,
+    ),
 )
 
 
@@ -554,6 +561,32 @@ def run_eval(rubric_id: str, trace_id: str) -> dict:
     return {"status": "ok", "rubric_id": rubric_id, "trace_id": trace_id, "result": _row(row)}
 
 
+# --- auth middleware --------------------------------------------------------
+
+class _BearerTokenMiddleware(BaseHTTPMiddleware):
+    """Enforce Bearer-token authentication on every MCP request.
+
+    Only active when a tunnel token has been configured via
+    ``spooling token generate`` or the ``SPOOLING_MCP_TOKEN`` env var.
+    Requests without a valid ``Authorization: Bearer <token>`` header receive
+    a 401 response and are never forwarded to the MCP handler.
+    """
+
+    def __init__(self, app, token: str) -> None:
+        super().__init__(app)
+        self._token = token
+
+    async def dispatch(self, request: Request, call_next):  # type: ignore[override]
+        auth = request.headers.get("Authorization", "")
+        if not (auth.startswith("Bearer ") and auth[7:].rstrip() == self._token):
+            return Response(
+                "Unauthorized",
+                status_code=401,
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+        return await call_next(request)
+
+
 # --- entrypoint ------------------------------------------------------------
 
 def serve_stdio() -> None:
@@ -562,8 +595,24 @@ def serve_stdio() -> None:
 
 
 def serve_http() -> None:
-    """Run the MCP server over streamable-HTTP at MCP_URL."""
-    mcp.run(transport="streamable-http")
+    """Run the MCP server over streamable-HTTP at MCP_URL.
+
+    If a tunnel token is configured (via ``spooling token generate`` or the
+    ``SPOOLING_MCP_TOKEN`` env var), every inbound request must carry a valid
+    ``Authorization: Bearer <token>`` header.  This keeps the public tunnel
+    endpoint private even though it's accessible over the internet.
+    """
+    from spooling.tunnel_auth import load_token
+
+    token = load_token()
+    if token:
+        import uvicorn
+
+        app = mcp.streamable_http_app()
+        app.add_middleware(_BearerTokenMiddleware, token=token)
+        uvicorn.run(app, host=MCP_HOST, port=MCP_PORT, log_level="warning")
+    else:
+        mcp.run(transport="streamable-http")
 
 
 if __name__ == "__main__":