spooling 0.1.4__tar.gz → 0.1.6__tar.gz

{spooling-0.1.4/spooling.egg-info → spooling-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spooling
-Version: 0.1.4
+Version: 0.1.6
 Summary: Local session tracker and semantic search for AI coding assistants
 Author: Parsed Analytics, Inc.
 License-Expression: MIT
@@ -26,6 +26,12 @@ Requires-Dist: mcp>=1.27
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: pytest-mock>=3.14; extra == "dev"
+Provides-Extra: connectors
+Requires-Dist: snowflake-connector-python>=3.0; extra == "connectors"
+Requires-Dist: google-cloud-bigquery>=3.0; extra == "connectors"
+Requires-Dist: mysql-connector-python>=8.0; extra == "connectors"
+Requires-Dist: clickhouse-connect>=0.5; extra == "connectors"
+Requires-Dist: motor>=3.0; extra == "connectors"
 Dynamic: license-file
 
 # Spooling
@@ -278,6 +284,22 @@ What it does each cycle:
 Cycles with no new work are silent. If a push fails the watermark is
 not advanced, so the next cycle retries the same window.
 
+### Cloud API endpoints (programmatic access)
+
+The cloud API at `api.spooling.ai` uses `/v1/` endpoints. Requests require a Bearer token via the `Authorization` header:
+
+```bash
+# Stats
+curl -s "https://api.spooling.ai/v1/stats" \
+  -H "Authorization: Bearer $SPOOLING_KEY"
+
+# Search (when deployed — falls back to local search on self-hosted)
+curl -s "https://api.spooling.ai/api/search?q=migration&limit=10" \
+  -H "Authorization: Bearer $SPOOLING_KEY"
+```
+
+**Important**: `app.spooling.ai` is the Next.js frontend. For API access, use `api.spooling.ai` directly or set `API_URL=https://api.spooling.ai` when deploying the frontend so its `/api/*` rewrites reach the backend.
+
 ### Status / logout
 
 ```bash
@@ -485,6 +507,7 @@ All optional - defaults work out of the box for local development.
 | `SPOOLING_EMBEDDING_MODEL` | `all-MiniLM-L6-v2` | Sentence transformer model |
 | `SPOOLING_UI_HOST` | `127.0.0.1` | API server host |
 | `ANTHROPIC_API_KEY` | - | Anthropic API key (alternative to setting in UI) |
+| `API_URL` | `http://127.0.0.1:3002` | Backend API URL for Next.js rewrites (`/api/*`→`<API_URL>/api/*`). Set to `https://api.spooling.ai` in production. |
 
 ---
 

{spooling-0.1.4 → spooling-0.1.6}/README.md

@@ -248,6 +248,22 @@ What it does each cycle:
 Cycles with no new work are silent. If a push fails the watermark is
 not advanced, so the next cycle retries the same window.
 
+### Cloud API endpoints (programmatic access)
+
+The cloud API at `api.spooling.ai` uses `/v1/` endpoints. Requests require a Bearer token via the `Authorization` header:
+
+```bash
+# Stats
+curl -s "https://api.spooling.ai/v1/stats" \
+  -H "Authorization: Bearer $SPOOLING_KEY"
+
+# Search (when deployed — falls back to local search on self-hosted)
+curl -s "https://api.spooling.ai/api/search?q=migration&limit=10" \
+  -H "Authorization: Bearer $SPOOLING_KEY"
+```
+
+**Important**: `app.spooling.ai` is the Next.js frontend. For API access, use `api.spooling.ai` directly or set `API_URL=https://api.spooling.ai` when deploying the frontend so its `/api/*` rewrites reach the backend.
+
 ### Status / logout
 
 ```bash
@@ -455,6 +471,7 @@ All optional - defaults work out of the box for local development.
 | `SPOOLING_EMBEDDING_MODEL` | `all-MiniLM-L6-v2` | Sentence transformer model |
 | `SPOOLING_UI_HOST` | `127.0.0.1` | API server host |
 | `ANTHROPIC_API_KEY` | - | Anthropic API key (alternative to setting in UI) |
+| `API_URL` | `http://127.0.0.1:3002` | Backend API URL for Next.js rewrites (`/api/*`→`<API_URL>/api/*`). Set to `https://api.spooling.ai` in production. |
 
 ---
 

{spooling-0.1.4 → spooling-0.1.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "spooling"
-version = "0.1.4"
+version = "0.1.6"
 description = "Local session tracker and semantic search for AI coding assistants"
 readme = "README.md"
 license = "MIT"
@@ -39,6 +39,13 @@ dev = [
     "pytest>=8.0",
     "pytest-mock>=3.14",
 ]
+connectors = [
+    "snowflake-connector-python>=3.0",
+    "google-cloud-bigquery>=3.0",
+    "mysql-connector-python>=8.0",
+    "clickhouse-connect>=0.5",
+    "motor>=3.0",
+]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

{spooling-0.1.4 → spooling-0.1.6}/spooling/cli.py

@@ -83,17 +83,22 @@ def watch():
 @click.argument("query")
 @click.option("-n", "--limit", default=10, help="Number of results")
 @click.option("-p", "--project", default=None, help="Filter by project")
-def search(query, limit, project):
+@click.option("--cloud", "cloud_mode", is_flag=True, help="Search cloud workspace instead of local DB")
+def search(query, limit, project, cloud_mode):
     """Semantic search across session history."""
-    from spooling.search import search as do_search
-
-    results = do_search(query, limit=limit, project=project)
+    if cloud_mode:
+        from spooling.cloud import cloud_search
+        results = cloud_search(query, limit=limit, project=project)
+    else:
+        from spooling.search import search as do_search
+        results = do_search(query, limit=limit, project=project)
 
     if not results:
         console.print("[yellow]No results found.[/yellow]")
         return
 
     for i, r in enumerate(results, 1):
+        source = r.get("_source", "local")
         similarity = f"{r['similarity']:.1%}"
         project_name = r["project"] or "unknown"
         role = r["role"]
@@ -104,6 +109,7 @@ def search(query, limit, project):
             f"[dim]{project_name}[/dim] "
             f"[{'green' if role == 'user' else 'blue'}]{role}[/{'green' if role == 'user' else 'blue'}] "
             f"[dim]{ts[:19]}[/dim]"
+            + (f" [cyan](cloud)[/cyan]" if source == "cloud" else "")
         )
         if r["title"]:
             console.print(f"   [dim]Session:[/dim] {r['title']}")
@@ -113,8 +119,23 @@ def search(query, limit, project):
 @cli.command()
 @click.option("--week", is_flag=True, help="Show weekly breakdown")
 @click.option("--days", default=7, help="Number of days for daily stats")
-def stats(week, days):
+@click.option("--cloud", "cloud_mode", is_flag=True, help="Show stats from cloud workspace")
+def stats(week, days, cloud_mode):
     """Show usage statistics."""
+    if cloud_mode:
+        from spooling.cloud import cloud_stats
+        data = cloud_stats()
+        if not data:
+            console.print("[yellow]No cloud stats available. Run 'spooling cloud login' first.[/yellow]")
+            return
+        console.print(Panel(
+            f"Sessions: [bold]{data.get('sessions', 0)}[/bold]  |  "
+            f"Cost: [bold]${float(data.get('cost', 0)):.2f}[/bold] est.",
+            title="[bold]Spooling Cloud Overview[/bold]",
+            style="cyan",
+        ))
+        return
+
     from spooling.stats import get_overview, get_daily_stats
 
     overview = get_overview()
@@ -659,21 +680,30 @@ def pricing_show(model):
 
 @cli.command()
 @click.option("--stdio", is_flag=True, help="Use stdio transport (default is streamable-HTTP)")
-def mcp(stdio):
+@click.option("--local", "local_mode", flag_value="local", default=True, help="Local DB only, no cloud fallback")
+@click.option("--cloud", "local_mode", flag_value="cloud", help="Cloud only, skip local DB")
+@click.option("--hybrid", "local_mode", flag_value="hybrid", help="Local DB with cloud fallback (default)")
+def mcp(stdio, local_mode):
     """Launch the Spooling MCP server.
 
     Defaults to streamable-HTTP at http://127.0.0.1:3004/mcp so any
     MCP-compatible agent (Codex, Cursor, web agents) can
     connect by URL. Pass --stdio for stdio-only clients.
+
+    Data source modes:
+    \b
+    --hybrid   Local DB with cloud fallback (default)
+    --local    Local DB only
+    --cloud    Cloud only
     """
+    from spooling.mcp_server import set_mode, serve_http as _serve_http, serve_stdio as _serve_stdio, MCP_URL
+    set_mode(local_mode or "hybrid")
     if stdio:
-        from spooling.mcp_server import serve_stdio
         console.print("[bold]Spooling MCP[/bold] over stdio")
-        serve_stdio()
+        _serve_stdio()
     else:
-        from spooling.mcp_server import serve_http, MCP_URL
         console.print(f"[bold]Spooling MCP[/bold] at {MCP_URL}")
-        serve_http()
+        _serve_http()
 
 
 def _check_ollama_preflight() -> None:

{spooling-0.1.4 → spooling-0.1.6}/spooling/cloud.py

@@ -766,3 +766,362 @@ def cloud_delete(project: str | None, cwd_substr: str | None, session_id: str |
         return
 
     console.print(f"[green]Deleted[/green] {deleted} session(s) from {base}")
+
+
+# --- Pull (cloud → local) ---------------------------------------------------
+
+
+def _store_cloud_session_locally(
+    session: dict,
+    messages: list[dict],
+) -> str | None:
+    """Upsert a cloud session + its messages into the local Spooling DB.
+
+    Returns the session id on success, ``None`` on DB error.
+    """
+    from datetime import datetime as _dt
+
+    conn = get_connection()
+    try:
+        sid = session["id"]
+        started = _dt.fromisoformat(session["started_at"]) if session.get("started_at") else None
+        ended = _dt.fromisoformat(session["ended_at"]) if session.get("ended_at") else None
+
+        conn.execute(
+            """INSERT INTO sessions (
+                id, provider_id, project, title, cwd,
+                started_at, ended_at, message_count, tool_call_count,
+                estimated_input_tokens, estimated_output_tokens, estimated_cost_usd, model
+            ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+            ON CONFLICT (id) DO UPDATE SET
+                provider_id = EXCLUDED.provider_id,
+                project = EXCLUDED.project,
+                title = EXCLUDED.title,
+                cwd = EXCLUDED.cwd,
+                started_at = EXCLUDED.started_at,
+                ended_at = EXCLUDED.ended_at,
+                message_count = EXCLUDED.message_count,
+                tool_call_count = EXCLUDED.tool_call_count,
+                estimated_input_tokens = EXCLUDED.estimated_input_tokens,
+                estimated_output_tokens = EXCLUDED.estimated_output_tokens,
+                estimated_cost_usd = EXCLUDED.estimated_cost_usd,
+                model = EXCLUDED.model""",
+            (
+                sid,
+                session.get("provider_id", "cloud"),
+                session.get("project"),
+                session.get("title"),
+                session.get("cwd"),
+                started,
+                ended,
+                session.get("message_count", len(messages)),
+                session.get("tool_call_count", 0),
+                session.get("input_tokens", 0),
+                session.get("output_tokens", 0),
+                session.get("estimated_cost_usd", 0.0),
+                session.get("model"),
+            ),
+        )
+
+        if messages:
+            for m in messages:
+                msg_id = f"{sid}-{m.get('sequence', 0)}"
+                ts = _dt.fromisoformat(m["timestamp"]) if m.get("timestamp") else None
+                conn.execute(
+                    """INSERT INTO messages (id, session_id, role, content, timestamp)
+                       VALUES (%s, %s, %s, %s, %s)
+                       ON CONFLICT (id) DO NOTHING""",
+                    (msg_id, sid, m["role"], m.get("content", ""), ts),
+                )
+        conn.commit()
+        return sid
+    except Exception as exc:
+        console.print(f"[red]  Error storing session {session.get('id', '?')}: {exc}[/red]")
+        conn.rollback()
+        return None
+    finally:
+        conn.close()
+
+
+def _pull_sessions(
+    limit: int,
+    batch: int,
+    base: str,
+    headers: dict,
+    since: datetime | None,
+    log,
+) -> tuple[int, int, str | None]:
+    """Fetch sessions from the cloud and store them locally.
+
+    Returns ``(accepted, skipped, error)``.
+    """
+    params = {"limit": min(limit, 200), "offset": 0}
+    if since:
+        params["since"] = since.isoformat()
+
+    with httpx.Client(timeout=60) as client:
+        try:
+            r = client.get(f"{base}/v1/sessions", headers=headers, params=params)
+            r.raise_for_status()
+        except httpx.HTTPStatusError as e:
+            return 0, 0, f"HTTP {e.response.status_code}: {e.response.text[:200]}"
+        except Exception as e:
+            return 0, 0, str(e)
+
+        data = r.json()
+        cloud_sessions = data.get("sessions", [])
+        if not cloud_sessions:
+            return 0, 0, None
+
+        accepted = 0
+        skipped = 0
+        for s in cloud_sessions:
+            try:
+                detail_r = client.get(
+                    f"{base}/v1/sessions/{s['id']}",
+                    headers=headers,
+                    timeout=30,
+                )
+                if not detail_r.is_success:
+                    skipped += 1
+                    log(f"  [yellow]skipped {s['id'][:12]}… ({detail_r.status_code})[/yellow]")
+                    continue
+                detail = detail_r.json()
+                result = _store_cloud_session_locally(
+                    detail.get("session", s),
+                    detail.get("messages", []),
+                )
+                if result:
+                    accepted += 1
+                else:
+                    skipped += 1
+            except Exception as e:
+                skipped += 1
+                log(f"  [yellow]error pulling {s.get('id', '?')[:12]}…: {e}[/yellow]")
+
+    return accepted, skipped, None
+
+
+@cloud.command("pull")
+@click.option("--limit", default=100, help="Max sessions to pull per run")
+@click.option("--batch", default=10, help="Concurrent session detail fetches")
+@click.option("--dry-run", is_flag=True, help="Show which sessions would be pulled. No writes.")
+def pull(limit: int, batch: int, dry_run: bool):
+    """Pull sessions from Spooling Cloud into the local database.
+
+    Reads the last-pull watermark from ``~/.config/spooling/cloud.json``
+    and fetches sessions synced since then. Combine with ``spooling push``
+    for a full bidirectional sync.
+    """
+    headers = _auth_headers()
+    base = _api_base()
+
+    cfg = _load_config()
+    last = cfg.get("last_pull_at")
+    watermark: datetime | None = datetime.fromisoformat(last) if last else None
+
+    if watermark:
+        console.print(f"[dim]Last pull: {watermark.strftime('%Y-%m-%d %H:%M:%S UTC')}[/dim]")
+
+    # First fetch the list to show the preview.
+    params = {"limit": min(limit, 200), "offset": 0}
+    if watermark:
+        params["since"] = watermark.isoformat()
+
+    try:
+        r = httpx.get(f"{base}/v1/sessions", headers=headers, params=params, timeout=30)
+        r.raise_for_status()
+    except httpx.HTTPStatusError as e:
+        console.print(f"[red]HTTP {e.response.status_code}: {e.response.text[:200]}[/red]")
+        return
+    except Exception as e:
+        console.print(f"[red]{e}[/red]")
+        return
+
+    data = r.json()
+    cloud_sessions = data.get("sessions", [])
+    if not cloud_sessions:
+        console.print("[yellow]No new cloud sessions to pull.[/yellow]")
+        return
+
+    if dry_run:
+        console.print(
+            f"[dim]Dry run: {len(cloud_sessions)} session(s) would be "
+            f"pulled from {base}[/dim]"
+        )
+        for s in cloud_sessions[:20]:
+            title = (s.get("title") or "(untitled)")[:60]
+            by = s.get("pushed_by_name") or s.get("pushed_by_email") or "?"
+            console.print(
+                f"  [cyan]{s.get('project') or '-'}[/cyan]  {title}  "
+                f"[dim](by {by})[/dim]"
+            )
+        if len(cloud_sessions) > 20:
+            console.print(f"  ... and {len(cloud_sessions) - 20} more")
+        return
+
+    accepted, skipped, err = _pull_sessions(limit, batch, base, headers, watermark, console.print)
+    if err:
+        console.print(f"[red]{err}[/red]")
+        return
+
+    # Advance watermark
+    new_watermark = datetime.now(timezone.utc)
+    cfg = _load_config()
+    cfg["last_pull_at"] = new_watermark.isoformat()
+    _save_config(cfg)
+
+    console.print(
+        f"[green]Done.[/green] {accepted} session(s) pulled from {base}"
+        + (f" ([yellow]{skipped} skipped[/yellow])" if skipped else "")
+    )
+
+
+@cloud.command("sync")
+@click.option("--limit", default=100, help="Max sessions per direction")
+@click.option("--batch", default=20, help="Sessions per request")
+@click.option(
+    "--project", default=None,
+    help="Only sync sessions whose project matches exactly.",
+)
+@click.option(
+    "--cwd", "cwd_substr", default=None,
+    help="Only sync sessions whose cwd contains this substring.",
+)
+@click.option(
+    "--title", "title_substr", default=None,
+    help="Only sync sessions whose title contains this substring.",
+)
+@click.option("--dry-run", is_flag=True, help="Show what would be synced. No network writes.")
+@click.option(
+    "--no-redact", "no_redact", is_flag=True,
+    help="Skip the client-side secret redactor.",
+)
+def sync(limit: int, batch: int, project: str | None, cwd_substr: str | None, title_substr: str | None, dry_run: bool, no_redact: bool):
+    """Bidirectional sync: push local sessions up, then pull cloud sessions down.
+
+    Runs ``spooling push`` first, then ``spooling pull``, so after this
+    command completes the local DB and the cloud workspace have the same
+    sessions (subject to filter scoping).
+    """
+    headers = _auth_headers()
+    base = _api_base()
+
+    # Phase 1: Push local → cloud
+    console.print("[bold]Phase 1: Push local → cloud[/bold]")
+    local_sessions = _collect_sessions(
+        limit=limit,
+        since=None,
+        project=project,
+        cwd_substr=cwd_substr,
+        title_substr=title_substr,
+    )
+    if not local_sessions:
+        console.print("  [yellow]No local sessions to push.[/yellow]")
+    else:
+        if not no_redact:
+            total_redactions = 0
+            for s in local_sessions:
+                _, n = redact_messages(s.get("messages") or [])
+                total_redactions += n
+            if total_redactions:
+                console.print(f"  [dim]Redacted {total_redactions} secret(s) before push.[/dim]")
+
+        if dry_run:
+            console.print(f"  [dim]Dry run: {len(local_sessions)} session(s) would be pushed[/dim]")
+        else:
+            pushed, rejected, err = _push_batches(local_sessions, batch, base, headers, console.print)
+            if err:
+                console.print(f"  [red]{err}[/red]")
+            else:
+                note = f" ([yellow]{rejected} rejected[/yellow])" if rejected else ""
+                console.print(f"  [green]Pushed[/green] {pushed} session(s){note}")
+
+    # Phase 2: Pull cloud → local
+    console.print("[bold]Phase 2: Pull cloud → local[/bold]")
+    cfg = _load_config()
+    last = cfg.get("last_pull_at")
+    watermark: datetime | None = datetime.fromisoformat(last) if last else None
+
+    params = {"limit": min(limit, 200), "offset": 0}
+    if watermark:
+        params["since"] = watermark.isoformat()
+
+    try:
+        r = httpx.get(f"{base}/v1/sessions", headers=headers, params=params, timeout=30)
+        r.raise_for_status()
+    except httpx.HTTPStatusError as e:
+        console.print(f"  [red]HTTP {e.response.status_code}: {e.response.text[:200]}[/red]")
+        return
+    except Exception as e:
+        console.print(f"  [red]{e}[/red]")
+        return
+
+    data = r.json()
+    cloud_sessions = data.get("sessions", [])
+    if not cloud_sessions:
+        console.print("  [yellow]No new cloud sessions to pull.[/yellow]")
+    else:
+        if dry_run:
+            console.print(f"  [dim]Dry run: {len(cloud_sessions)} session(s) would be pulled[/dim]")
+            for s in cloud_sessions[:10]:
+                title = (s.get("title") or "(untitled)")[:60]
+                by = s.get("pushed_by_name") or s.get("pushed_by_email") or "?"
+                console.print(f"    [cyan]{s.get('project') or '-'}[/cyan]  {title}  [dim](by {by})[/dim]")
+        else:
+            accepted, skipped, err = _pull_sessions(limit, batch, base, headers, watermark, lambda msg: console.print(f"  {msg}"))
+            if err:
+                console.print(f"  [red]{err}[/red]")
+            else:
+                new_watermark = datetime.now(timezone.utc)
+                cfg = _load_config()
+                cfg["last_pull_at"] = new_watermark.isoformat()
+                _save_config(cfg)
+                skip_note = f" ([yellow]{skipped} skipped[/yellow])" if skipped else ""
+                console.print(f"  [green]Pulled[/green] {accepted} session(s){skip_note}")
+
+    console.print("[bold green]Bidirectional sync complete.[/bold green]")
+
+
+# --- Cloud helpers for --cloud mode on search / stats / mcp ------------------
+
+def cloud_search(query: str, limit: int = 10, project: str | None = None) -> list[dict]:
+    """Search Spooling Cloud via ILIKE. Returns list of result dicts."""
+    headers = _auth_headers()
+    base = _api_base()
+    params: dict = {"q": query, "limit": limit}
+    if project:
+        params["project"] = project
+    try:
+        r = httpx.get(f"{base}/v1/search", headers=headers, params=params, timeout=15)
+        r.raise_for_status()
+        data = r.json()
+    except Exception:
+        return []
+    sessions = data.get("sessions", [])
+    return [
+        {
+            "content": f"{s.get('title') or '(untitled)'} — {s.get('project') or '?'}",
+            "role": "system",
+            "project": s.get("project"),
+            "timestamp": s.get("started_at"),
+            "session_id": s["id"],
+            "similarity": 0.0,
+            "title": s.get("title"),
+            "cwd": None,
+            "_source": "cloud",
+        }
+        for s in sessions
+    ]
+
+
+def cloud_stats() -> dict | None:
+    """Fetch workspace-level stats from Spooling Cloud. Returns dict or None."""
+    headers = _auth_headers()
+    base = _api_base()
+    try:
+        r = httpx.get(f"{base}/v1/stats", headers=headers, timeout=15)
+        r.raise_for_status()
+        return r.json()
+    except Exception:
+        return None

spooling-0.1.6/spooling/connectors/base.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from .types import (
+    ColumnInfo,
+    ConnectionConfig,
+    QueryResult,
+    TableInfo,
+    TestConnectionResult,
+)
+
+
+class DataConnector(ABC):
+    @abstractmethod
+    async def connect(self) -> None: ...
+
+    @abstractmethod
+    async def disconnect(self) -> None: ...
+
+    @abstractmethod
+    def is_connected(self) -> bool: ...
+
+    @abstractmethod
+    async def test_connection(self) -> TestConnectionResult: ...
+
+    @abstractmethod
+    async def query(self, sql: str, params: list | None = None) -> QueryResult: ...
+
+    @abstractmethod
+    async def list_schemas(self) -> list[str]: ...
+
+    @abstractmethod
+    async def list_tables(self, schema: str | None = None) -> list[TableInfo]: ...
+
+    @abstractmethod
+    async def get_table_columns(self, schema: str, table: str) -> list[ColumnInfo]: ...