spooling 0.1.5__tar.gz → 0.1.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {spooling-0.1.5/spooling.egg-info → spooling-0.1.6}/PKG-INFO +7 -1
  2. {spooling-0.1.5 → spooling-0.1.6}/pyproject.toml +8 -1
  3. {spooling-0.1.5 → spooling-0.1.6}/spooling/cli.py +40 -10
  4. {spooling-0.1.5 → spooling-0.1.6}/spooling/cloud.py +359 -0
  5. spooling-0.1.6/spooling/connectors/__init__.py +0 -0
  6. spooling-0.1.6/spooling/connectors/base.py +37 -0
  7. spooling-0.1.6/spooling/connectors/bigquery.py +140 -0
  8. spooling-0.1.6/spooling/connectors/clickhouse.py +111 -0
  9. spooling-0.1.6/spooling/connectors/factory.py +68 -0
  10. spooling-0.1.6/spooling/connectors/mongodb.py +103 -0
  11. spooling-0.1.6/spooling/connectors/mysql.py +129 -0
  12. spooling-0.1.6/spooling/connectors/postgresql.py +120 -0
  13. spooling-0.1.6/spooling/connectors/rest/__init__.py +0 -0
  14. spooling-0.1.6/spooling/connectors/rest/base.py +206 -0
  15. spooling-0.1.6/spooling/connectors/rest/shopify_connectors.py +172 -0
  16. spooling-0.1.6/spooling/connectors/snowflake.py +155 -0
  17. spooling-0.1.6/spooling/connectors/types.py +87 -0
  18. {spooling-0.1.5 → spooling-0.1.6}/spooling/mcp_server.py +266 -8
  19. {spooling-0.1.5 → spooling-0.1.6}/spooling/server.py +326 -0
  20. {spooling-0.1.5 → spooling-0.1.6/spooling.egg-info}/PKG-INFO +7 -1
  21. {spooling-0.1.5 → spooling-0.1.6}/spooling.egg-info/SOURCES.txt +13 -0
  22. {spooling-0.1.5 → spooling-0.1.6}/spooling.egg-info/requires.txt +7 -0
  23. {spooling-0.1.5 → spooling-0.1.6}/LICENSE +0 -0
  24. {spooling-0.1.5 → spooling-0.1.6}/README.md +0 -0
  25. {spooling-0.1.5 → spooling-0.1.6}/setup.cfg +0 -0
  26. {spooling-0.1.5 → spooling-0.1.6}/spooling/__init__.py +0 -0
  27. {spooling-0.1.5 → spooling-0.1.6}/spooling/agent.py +0 -0
  28. {spooling-0.1.5 → spooling-0.1.6}/spooling/classifiers.py +0 -0
  29. {spooling-0.1.5 → spooling-0.1.6}/spooling/config.py +0 -0
  30. {spooling-0.1.5 → spooling-0.1.6}/spooling/db.py +0 -0
  31. {spooling-0.1.5 → spooling-0.1.6}/spooling/embeddings.py +0 -0
  32. {spooling-0.1.5 → spooling-0.1.6}/spooling/evals.py +0 -0
  33. {spooling-0.1.5 → spooling-0.1.6}/spooling/experiments.py +0 -0
  34. {spooling-0.1.5 → spooling-0.1.6}/spooling/ingest.py +0 -0
  35. {spooling-0.1.5 → spooling-0.1.6}/spooling/parser.py +0 -0
  36. {spooling-0.1.5 → spooling-0.1.6}/spooling/pricing.py +0 -0
  37. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/__init__.py +0 -0
  38. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/antigravity.py +0 -0
  39. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/base.py +0 -0
  40. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/codex.py +0 -0
  41. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/copilot.py +0 -0
  42. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/cortex_code.py +0 -0
  43. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/cursor.py +0 -0
  44. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/gemini.py +0 -0
  45. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/github.py +0 -0
  46. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/gitlab.py +0 -0
  47. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/kiro.py +0 -0
  48. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/opencode.py +0 -0
  49. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/session_file.py +0 -0
  50. {spooling-0.1.5 → spooling-0.1.6}/spooling/providers/windsurf.py +0 -0
  51. {spooling-0.1.5 → spooling-0.1.6}/spooling/redact.py +0 -0
  52. {spooling-0.1.5 → spooling-0.1.6}/spooling/remote_otel.py +0 -0
  53. {spooling-0.1.5 → spooling-0.1.6}/spooling/sdk.py +0 -0
  54. {spooling-0.1.5 → spooling-0.1.6}/spooling/search.py +0 -0
  55. {spooling-0.1.5 → spooling-0.1.6}/spooling/stats.py +0 -0
  56. {spooling-0.1.5 → spooling-0.1.6}/spooling/subscription_pricing.py +0 -0
  57. {spooling-0.1.5 → spooling-0.1.6}/spooling/tracing.py +0 -0
  58. {spooling-0.1.5 → spooling-0.1.6}/spooling/watcher.py +0 -0
  59. {spooling-0.1.5 → spooling-0.1.6}/spooling.egg-info/dependency_links.txt +0 -0
  60. {spooling-0.1.5 → spooling-0.1.6}/spooling.egg-info/entry_points.txt +0 -0
  61. {spooling-0.1.5 → spooling-0.1.6}/spooling.egg-info/top_level.txt +0 -0
  62. {spooling-0.1.5 → spooling-0.1.6}/tests/test_parser.py +0 -0
  63. {spooling-0.1.5 → spooling-0.1.6}/tests/test_pricing.py +0 -0
  64. {spooling-0.1.5 → spooling-0.1.6}/tests/test_redact.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: spooling
3
- Version: 0.1.5
3
+ Version: 0.1.6
4
4
  Summary: Local session tracker and semantic search for AI coding assistants
5
5
  Author: Parsed Analytics, Inc.
6
6
  License-Expression: MIT
@@ -26,6 +26,12 @@ Requires-Dist: mcp>=1.27
26
26
  Provides-Extra: dev
27
27
  Requires-Dist: pytest>=8.0; extra == "dev"
28
28
  Requires-Dist: pytest-mock>=3.14; extra == "dev"
29
+ Provides-Extra: connectors
30
+ Requires-Dist: snowflake-connector-python>=3.0; extra == "connectors"
31
+ Requires-Dist: google-cloud-bigquery>=3.0; extra == "connectors"
32
+ Requires-Dist: mysql-connector-python>=8.0; extra == "connectors"
33
+ Requires-Dist: clickhouse-connect>=0.5; extra == "connectors"
34
+ Requires-Dist: motor>=3.0; extra == "connectors"
29
35
  Dynamic: license-file
30
36
 
31
37
  # Spooling
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "spooling"
3
- version = "0.1.5"
3
+ version = "0.1.6"
4
4
  description = "Local session tracker and semantic search for AI coding assistants"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -39,6 +39,13 @@ dev = [
39
39
  "pytest>=8.0",
40
40
  "pytest-mock>=3.14",
41
41
  ]
42
+ connectors = [
43
+ "snowflake-connector-python>=3.0",
44
+ "google-cloud-bigquery>=3.0",
45
+ "mysql-connector-python>=8.0",
46
+ "clickhouse-connect>=0.5",
47
+ "motor>=3.0",
48
+ ]
42
49
 
43
50
  [tool.pytest.ini_options]
44
51
  testpaths = ["tests"]
@@ -83,17 +83,22 @@ def watch():
83
83
  @click.argument("query")
84
84
  @click.option("-n", "--limit", default=10, help="Number of results")
85
85
  @click.option("-p", "--project", default=None, help="Filter by project")
86
- def search(query, limit, project):
86
+ @click.option("--cloud", "cloud_mode", is_flag=True, help="Search cloud workspace instead of local DB")
87
+ def search(query, limit, project, cloud_mode):
87
88
  """Semantic search across session history."""
88
- from spooling.search import search as do_search
89
-
90
- results = do_search(query, limit=limit, project=project)
89
+ if cloud_mode:
90
+ from spooling.cloud import cloud_search
91
+ results = cloud_search(query, limit=limit, project=project)
92
+ else:
93
+ from spooling.search import search as do_search
94
+ results = do_search(query, limit=limit, project=project)
91
95
 
92
96
  if not results:
93
97
  console.print("[yellow]No results found.[/yellow]")
94
98
  return
95
99
 
96
100
  for i, r in enumerate(results, 1):
101
+ source = r.get("_source", "local")
97
102
  similarity = f"{r['similarity']:.1%}"
98
103
  project_name = r["project"] or "unknown"
99
104
  role = r["role"]
@@ -104,6 +109,7 @@ def search(query, limit, project):
104
109
  f"[dim]{project_name}[/dim] "
105
110
  f"[{'green' if role == 'user' else 'blue'}]{role}[/{'green' if role == 'user' else 'blue'}] "
106
111
  f"[dim]{ts[:19]}[/dim]"
112
+ + (f" [cyan](cloud)[/cyan]" if source == "cloud" else "")
107
113
  )
108
114
  if r["title"]:
109
115
  console.print(f" [dim]Session:[/dim] {r['title']}")
@@ -113,8 +119,23 @@ def search(query, limit, project):
113
119
  @cli.command()
114
120
  @click.option("--week", is_flag=True, help="Show weekly breakdown")
115
121
  @click.option("--days", default=7, help="Number of days for daily stats")
116
- def stats(week, days):
122
+ @click.option("--cloud", "cloud_mode", is_flag=True, help="Show stats from cloud workspace")
123
+ def stats(week, days, cloud_mode):
117
124
  """Show usage statistics."""
125
+ if cloud_mode:
126
+ from spooling.cloud import cloud_stats
127
+ data = cloud_stats()
128
+ if not data:
129
+ console.print("[yellow]No cloud stats available. Run 'spooling cloud login' first.[/yellow]")
130
+ return
131
+ console.print(Panel(
132
+ f"Sessions: [bold]{data.get('sessions', 0)}[/bold] | "
133
+ f"Cost: [bold]${float(data.get('cost', 0)):.2f}[/bold] est.",
134
+ title="[bold]Spooling Cloud Overview[/bold]",
135
+ style="cyan",
136
+ ))
137
+ return
138
+
118
139
  from spooling.stats import get_overview, get_daily_stats
119
140
 
120
141
  overview = get_overview()
@@ -659,21 +680,30 @@ def pricing_show(model):
659
680
 
660
681
  @cli.command()
661
682
  @click.option("--stdio", is_flag=True, help="Use stdio transport (default is streamable-HTTP)")
662
- def mcp(stdio):
683
+ @click.option("--local", "local_mode", flag_value="local", default=True, help="Local DB only, no cloud fallback")
684
+ @click.option("--cloud", "local_mode", flag_value="cloud", help="Cloud only, skip local DB")
685
+ @click.option("--hybrid", "local_mode", flag_value="hybrid", help="Local DB with cloud fallback (default)")
686
+ def mcp(stdio, local_mode):
663
687
  """Launch the Spooling MCP server.
664
688
 
665
689
  Defaults to streamable-HTTP at http://127.0.0.1:3004/mcp so any
666
690
  MCP-compatible agent (Codex, Cursor, web agents) can
667
691
  connect by URL. Pass --stdio for stdio-only clients.
692
+
693
+ Data source modes:
694
+ \b
695
+ --hybrid Local DB with cloud fallback (default)
696
+ --local Local DB only
697
+ --cloud Cloud only
668
698
  """
699
+ from spooling.mcp_server import set_mode, serve_http as _serve_http, serve_stdio as _serve_stdio, MCP_URL
700
+ set_mode(local_mode or "hybrid")
669
701
  if stdio:
670
- from spooling.mcp_server import serve_stdio
671
702
  console.print("[bold]Spooling MCP[/bold] over stdio")
672
- serve_stdio()
703
+ _serve_stdio()
673
704
  else:
674
- from spooling.mcp_server import serve_http, MCP_URL
675
705
  console.print(f"[bold]Spooling MCP[/bold] at {MCP_URL}")
676
- serve_http()
706
+ _serve_http()
677
707
 
678
708
 
679
709
  def _check_ollama_preflight() -> None:
@@ -766,3 +766,362 @@ def cloud_delete(project: str | None, cwd_substr: str | None, session_id: str |
766
766
  return
767
767
 
768
768
  console.print(f"[green]Deleted[/green] {deleted} session(s) from {base}")
769
+
770
+
771
+ # --- Pull (cloud → local) ---------------------------------------------------
772
+
773
+
774
+ def _store_cloud_session_locally(
775
+ session: dict,
776
+ messages: list[dict],
777
+ ) -> str | None:
778
+ """Upsert a cloud session + its messages into the local Spooling DB.
779
+
780
+ Returns the session id on success, ``None`` on DB error.
781
+ """
782
+ from datetime import datetime as _dt
783
+
784
+ conn = get_connection()
785
+ try:
786
+ sid = session["id"]
787
+ started = _dt.fromisoformat(session["started_at"]) if session.get("started_at") else None
788
+ ended = _dt.fromisoformat(session["ended_at"]) if session.get("ended_at") else None
789
+
790
+ conn.execute(
791
+ """INSERT INTO sessions (
792
+ id, provider_id, project, title, cwd,
793
+ started_at, ended_at, message_count, tool_call_count,
794
+ estimated_input_tokens, estimated_output_tokens, estimated_cost_usd, model
795
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
796
+ ON CONFLICT (id) DO UPDATE SET
797
+ provider_id = EXCLUDED.provider_id,
798
+ project = EXCLUDED.project,
799
+ title = EXCLUDED.title,
800
+ cwd = EXCLUDED.cwd,
801
+ started_at = EXCLUDED.started_at,
802
+ ended_at = EXCLUDED.ended_at,
803
+ message_count = EXCLUDED.message_count,
804
+ tool_call_count = EXCLUDED.tool_call_count,
805
+ estimated_input_tokens = EXCLUDED.estimated_input_tokens,
806
+ estimated_output_tokens = EXCLUDED.estimated_output_tokens,
807
+ estimated_cost_usd = EXCLUDED.estimated_cost_usd,
808
+ model = EXCLUDED.model""",
809
+ (
810
+ sid,
811
+ session.get("provider_id", "cloud"),
812
+ session.get("project"),
813
+ session.get("title"),
814
+ session.get("cwd"),
815
+ started,
816
+ ended,
817
+ session.get("message_count", len(messages)),
818
+ session.get("tool_call_count", 0),
819
+ session.get("input_tokens", 0),
820
+ session.get("output_tokens", 0),
821
+ session.get("estimated_cost_usd", 0.0),
822
+ session.get("model"),
823
+ ),
824
+ )
825
+
826
+ if messages:
827
+ for m in messages:
828
+ msg_id = f"{sid}-{m.get('sequence', 0)}"
829
+ ts = _dt.fromisoformat(m["timestamp"]) if m.get("timestamp") else None
830
+ conn.execute(
831
+ """INSERT INTO messages (id, session_id, role, content, timestamp)
832
+ VALUES (%s, %s, %s, %s, %s)
833
+ ON CONFLICT (id) DO NOTHING""",
834
+ (msg_id, sid, m["role"], m.get("content", ""), ts),
835
+ )
836
+ conn.commit()
837
+ return sid
838
+ except Exception as exc:
839
+ console.print(f"[red] Error storing session {session.get('id', '?')}: {exc}[/red]")
840
+ conn.rollback()
841
+ return None
842
+ finally:
843
+ conn.close()
844
+
845
+
846
+ def _pull_sessions(
847
+ limit: int,
848
+ batch: int,
849
+ base: str,
850
+ headers: dict,
851
+ since: datetime | None,
852
+ log,
853
+ ) -> tuple[int, int, str | None]:
854
+ """Fetch sessions from the cloud and store them locally.
855
+
856
+ Returns ``(accepted, skipped, error)``.
857
+ """
858
+ params = {"limit": min(limit, 200), "offset": 0}
859
+ if since:
860
+ params["since"] = since.isoformat()
861
+
862
+ with httpx.Client(timeout=60) as client:
863
+ try:
864
+ r = client.get(f"{base}/v1/sessions", headers=headers, params=params)
865
+ r.raise_for_status()
866
+ except httpx.HTTPStatusError as e:
867
+ return 0, 0, f"HTTP {e.response.status_code}: {e.response.text[:200]}"
868
+ except Exception as e:
869
+ return 0, 0, str(e)
870
+
871
+ data = r.json()
872
+ cloud_sessions = data.get("sessions", [])
873
+ if not cloud_sessions:
874
+ return 0, 0, None
875
+
876
+ accepted = 0
877
+ skipped = 0
878
+ for s in cloud_sessions:
879
+ try:
880
+ detail_r = client.get(
881
+ f"{base}/v1/sessions/{s['id']}",
882
+ headers=headers,
883
+ timeout=30,
884
+ )
885
+ if not detail_r.is_success:
886
+ skipped += 1
887
+ log(f" [yellow]skipped {s['id'][:12]}… ({detail_r.status_code})[/yellow]")
888
+ continue
889
+ detail = detail_r.json()
890
+ result = _store_cloud_session_locally(
891
+ detail.get("session", s),
892
+ detail.get("messages", []),
893
+ )
894
+ if result:
895
+ accepted += 1
896
+ else:
897
+ skipped += 1
898
+ except Exception as e:
899
+ skipped += 1
900
+ log(f" [yellow]error pulling {s.get('id', '?')[:12]}…: {e}[/yellow]")
901
+
902
+ return accepted, skipped, None
903
+
904
+
905
+ @cloud.command("pull")
906
+ @click.option("--limit", default=100, help="Max sessions to pull per run")
907
+ @click.option("--batch", default=10, help="Concurrent session detail fetches")
908
+ @click.option("--dry-run", is_flag=True, help="Show which sessions would be pulled. No writes.")
909
+ def pull(limit: int, batch: int, dry_run: bool):
910
+ """Pull sessions from Spooling Cloud into the local database.
911
+
912
+ Reads the last-pull watermark from ``~/.config/spooling/cloud.json``
913
+ and fetches sessions synced since then. Combine with ``spooling push``
914
+ for a full bidirectional sync.
915
+ """
916
+ headers = _auth_headers()
917
+ base = _api_base()
918
+
919
+ cfg = _load_config()
920
+ last = cfg.get("last_pull_at")
921
+ watermark: datetime | None = datetime.fromisoformat(last) if last else None
922
+
923
+ if watermark:
924
+ console.print(f"[dim]Last pull: {watermark.strftime('%Y-%m-%d %H:%M:%S UTC')}[/dim]")
925
+
926
+ # First fetch the list to show the preview.
927
+ params = {"limit": min(limit, 200), "offset": 0}
928
+ if watermark:
929
+ params["since"] = watermark.isoformat()
930
+
931
+ try:
932
+ r = httpx.get(f"{base}/v1/sessions", headers=headers, params=params, timeout=30)
933
+ r.raise_for_status()
934
+ except httpx.HTTPStatusError as e:
935
+ console.print(f"[red]HTTP {e.response.status_code}: {e.response.text[:200]}[/red]")
936
+ return
937
+ except Exception as e:
938
+ console.print(f"[red]{e}[/red]")
939
+ return
940
+
941
+ data = r.json()
942
+ cloud_sessions = data.get("sessions", [])
943
+ if not cloud_sessions:
944
+ console.print("[yellow]No new cloud sessions to pull.[/yellow]")
945
+ return
946
+
947
+ if dry_run:
948
+ console.print(
949
+ f"[dim]Dry run: {len(cloud_sessions)} session(s) would be "
950
+ f"pulled from {base}[/dim]"
951
+ )
952
+ for s in cloud_sessions[:20]:
953
+ title = (s.get("title") or "(untitled)")[:60]
954
+ by = s.get("pushed_by_name") or s.get("pushed_by_email") or "?"
955
+ console.print(
956
+ f" [cyan]{s.get('project') or '-'}[/cyan] {title} "
957
+ f"[dim](by {by})[/dim]"
958
+ )
959
+ if len(cloud_sessions) > 20:
960
+ console.print(f" ... and {len(cloud_sessions) - 20} more")
961
+ return
962
+
963
+ accepted, skipped, err = _pull_sessions(limit, batch, base, headers, watermark, console.print)
964
+ if err:
965
+ console.print(f"[red]{err}[/red]")
966
+ return
967
+
968
+ # Advance watermark
969
+ new_watermark = datetime.now(timezone.utc)
970
+ cfg = _load_config()
971
+ cfg["last_pull_at"] = new_watermark.isoformat()
972
+ _save_config(cfg)
973
+
974
+ console.print(
975
+ f"[green]Done.[/green] {accepted} session(s) pulled from {base}"
976
+ + (f" ([yellow]{skipped} skipped[/yellow])" if skipped else "")
977
+ )
978
+
979
+
980
+ @cloud.command("sync")
981
+ @click.option("--limit", default=100, help="Max sessions per direction")
982
+ @click.option("--batch", default=20, help="Sessions per request")
983
+ @click.option(
984
+ "--project", default=None,
985
+ help="Only sync sessions whose project matches exactly.",
986
+ )
987
+ @click.option(
988
+ "--cwd", "cwd_substr", default=None,
989
+ help="Only sync sessions whose cwd contains this substring.",
990
+ )
991
+ @click.option(
992
+ "--title", "title_substr", default=None,
993
+ help="Only sync sessions whose title contains this substring.",
994
+ )
995
+ @click.option("--dry-run", is_flag=True, help="Show what would be synced. No network writes.")
996
+ @click.option(
997
+ "--no-redact", "no_redact", is_flag=True,
998
+ help="Skip the client-side secret redactor.",
999
+ )
1000
+ def sync(limit: int, batch: int, project: str | None, cwd_substr: str | None, title_substr: str | None, dry_run: bool, no_redact: bool):
1001
+ """Bidirectional sync: push local sessions up, then pull cloud sessions down.
1002
+
1003
+ Runs ``spooling push`` first, then ``spooling pull``, so after this
1004
+ command completes the local DB and the cloud workspace have the same
1005
+ sessions (subject to filter scoping).
1006
+ """
1007
+ headers = _auth_headers()
1008
+ base = _api_base()
1009
+
1010
+ # Phase 1: Push local → cloud
1011
+ console.print("[bold]Phase 1: Push local → cloud[/bold]")
1012
+ local_sessions = _collect_sessions(
1013
+ limit=limit,
1014
+ since=None,
1015
+ project=project,
1016
+ cwd_substr=cwd_substr,
1017
+ title_substr=title_substr,
1018
+ )
1019
+ if not local_sessions:
1020
+ console.print(" [yellow]No local sessions to push.[/yellow]")
1021
+ else:
1022
+ if not no_redact:
1023
+ total_redactions = 0
1024
+ for s in local_sessions:
1025
+ _, n = redact_messages(s.get("messages") or [])
1026
+ total_redactions += n
1027
+ if total_redactions:
1028
+ console.print(f" [dim]Redacted {total_redactions} secret(s) before push.[/dim]")
1029
+
1030
+ if dry_run:
1031
+ console.print(f" [dim]Dry run: {len(local_sessions)} session(s) would be pushed[/dim]")
1032
+ else:
1033
+ pushed, rejected, err = _push_batches(local_sessions, batch, base, headers, console.print)
1034
+ if err:
1035
+ console.print(f" [red]{err}[/red]")
1036
+ else:
1037
+ note = f" ([yellow]{rejected} rejected[/yellow])" if rejected else ""
1038
+ console.print(f" [green]Pushed[/green] {pushed} session(s){note}")
1039
+
1040
+ # Phase 2: Pull cloud → local
1041
+ console.print("[bold]Phase 2: Pull cloud → local[/bold]")
1042
+ cfg = _load_config()
1043
+ last = cfg.get("last_pull_at")
1044
+ watermark: datetime | None = datetime.fromisoformat(last) if last else None
1045
+
1046
+ params = {"limit": min(limit, 200), "offset": 0}
1047
+ if watermark:
1048
+ params["since"] = watermark.isoformat()
1049
+
1050
+ try:
1051
+ r = httpx.get(f"{base}/v1/sessions", headers=headers, params=params, timeout=30)
1052
+ r.raise_for_status()
1053
+ except httpx.HTTPStatusError as e:
1054
+ console.print(f" [red]HTTP {e.response.status_code}: {e.response.text[:200]}[/red]")
1055
+ return
1056
+ except Exception as e:
1057
+ console.print(f" [red]{e}[/red]")
1058
+ return
1059
+
1060
+ data = r.json()
1061
+ cloud_sessions = data.get("sessions", [])
1062
+ if not cloud_sessions:
1063
+ console.print(" [yellow]No new cloud sessions to pull.[/yellow]")
1064
+ else:
1065
+ if dry_run:
1066
+ console.print(f" [dim]Dry run: {len(cloud_sessions)} session(s) would be pulled[/dim]")
1067
+ for s in cloud_sessions[:10]:
1068
+ title = (s.get("title") or "(untitled)")[:60]
1069
+ by = s.get("pushed_by_name") or s.get("pushed_by_email") or "?"
1070
+ console.print(f" [cyan]{s.get('project') or '-'}[/cyan] {title} [dim](by {by})[/dim]")
1071
+ else:
1072
+ accepted, skipped, err = _pull_sessions(limit, batch, base, headers, watermark, lambda msg: console.print(f" {msg}"))
1073
+ if err:
1074
+ console.print(f" [red]{err}[/red]")
1075
+ else:
1076
+ new_watermark = datetime.now(timezone.utc)
1077
+ cfg = _load_config()
1078
+ cfg["last_pull_at"] = new_watermark.isoformat()
1079
+ _save_config(cfg)
1080
+ skip_note = f" ([yellow]{skipped} skipped[/yellow])" if skipped else ""
1081
+ console.print(f" [green]Pulled[/green] {accepted} session(s){skip_note}")
1082
+
1083
+ console.print("[bold green]Bidirectional sync complete.[/bold green]")
1084
+
1085
+
1086
+ # --- Cloud helpers for --cloud mode on search / stats / mcp ------------------
1087
+
1088
+ def cloud_search(query: str, limit: int = 10, project: str | None = None) -> list[dict]:
1089
+ """Search Spooling Cloud via ILIKE. Returns list of result dicts."""
1090
+ headers = _auth_headers()
1091
+ base = _api_base()
1092
+ params: dict = {"q": query, "limit": limit}
1093
+ if project:
1094
+ params["project"] = project
1095
+ try:
1096
+ r = httpx.get(f"{base}/v1/search", headers=headers, params=params, timeout=15)
1097
+ r.raise_for_status()
1098
+ data = r.json()
1099
+ except Exception:
1100
+ return []
1101
+ sessions = data.get("sessions", [])
1102
+ return [
1103
+ {
1104
+ "content": f"{s.get('title') or '(untitled)'} — {s.get('project') or '?'}",
1105
+ "role": "system",
1106
+ "project": s.get("project"),
1107
+ "timestamp": s.get("started_at"),
1108
+ "session_id": s["id"],
1109
+ "similarity": 0.0,
1110
+ "title": s.get("title"),
1111
+ "cwd": None,
1112
+ "_source": "cloud",
1113
+ }
1114
+ for s in sessions
1115
+ ]
1116
+
1117
+
1118
+ def cloud_stats() -> dict | None:
1119
+ """Fetch workspace-level stats from Spooling Cloud. Returns dict or None."""
1120
+ headers = _auth_headers()
1121
+ base = _api_base()
1122
+ try:
1123
+ r = httpx.get(f"{base}/v1/stats", headers=headers, timeout=15)
1124
+ r.raise_for_status()
1125
+ return r.json()
1126
+ except Exception:
1127
+ return None
File without changes
@@ -0,0 +1,37 @@
1
+ from __future__ import annotations
2
+
3
+ from abc import ABC, abstractmethod
4
+
5
+ from .types import (
6
+ ColumnInfo,
7
+ ConnectionConfig,
8
+ QueryResult,
9
+ TableInfo,
10
+ TestConnectionResult,
11
+ )
12
+
13
+
14
+ class DataConnector(ABC):
15
+ @abstractmethod
16
+ async def connect(self) -> None: ...
17
+
18
+ @abstractmethod
19
+ async def disconnect(self) -> None: ...
20
+
21
+ @abstractmethod
22
+ def is_connected(self) -> bool: ...
23
+
24
+ @abstractmethod
25
+ async def test_connection(self) -> TestConnectionResult: ...
26
+
27
+ @abstractmethod
28
+ async def query(self, sql: str, params: list | None = None) -> QueryResult: ...
29
+
30
+ @abstractmethod
31
+ async def list_schemas(self) -> list[str]: ...
32
+
33
+ @abstractmethod
34
+ async def list_tables(self, schema: str | None = None) -> list[TableInfo]: ...
35
+
36
+ @abstractmethod
37
+ async def get_table_columns(self, schema: str, table: str) -> list[ColumnInfo]: ...
@@ -0,0 +1,140 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import time
5
+
6
+ import httpx
7
+
8
+ from spooling.connectors.base import DataConnector
9
+ from spooling.connectors.types import (
10
+ AuthenticationError,
11
+ ColumnInfo,
12
+ ConnectionConfig,
13
+ ConnectionError,
14
+ FieldInfo,
15
+ QueryError,
16
+ QueryResult,
17
+ TableInfo,
18
+ TestConnectionResult,
19
+ )
20
+
21
+ # BigQuery uses google-cloud-bigquery which is synchronous.
22
+ # We wrap in run_in_executor for async compatibility.
23
+
24
+
25
+ class BigQueryConnector(DataConnector):
26
+ def __init__(self, config: ConnectionConfig):
27
+ self.config = config
28
+ self._client = None
29
+
30
+ async def _get_client(self):
31
+ if self._client is None:
32
+ from google.cloud import bigquery
33
+ from google.oauth2 import service_account
34
+
35
+ if self.config.service_account:
36
+ credentials = service_account.Credentials.from_service_account_info(
37
+ self.config.service_account
38
+ )
39
+ self._client = bigquery.Client(
40
+ project=self.config.project_id or credentials.project_id,
41
+ credentials=credentials,
42
+ )
43
+ else:
44
+ self._client = bigquery.Client(project=self.config.project_id)
45
+ return self._client
46
+
47
+ async def connect(self) -> None:
48
+ import asyncio
49
+
50
+ try:
51
+ client = await self._get_client()
52
+ await asyncio.to_thread(client.query, "SELECT 1")
53
+ except Exception as e:
54
+ msg = str(e).lower()
55
+ if "unauthorized" in msg or "invalid" in msg or "denied" in msg:
56
+ raise AuthenticationError(f"BigQuery authentication failed: {e}") from e
57
+ raise ConnectionError(f"Failed to connect to BigQuery: {e}") from e
58
+
59
+ async def disconnect(self) -> None:
60
+ if self._client:
61
+ self._client.close()
62
+ self._client = None
63
+
64
+ def is_connected(self) -> bool:
65
+ return self._client is not None
66
+
67
+ async def test_connection(self) -> TestConnectionResult:
68
+ try:
69
+ if not self.is_connected():
70
+ await self.connect()
71
+ result = await self.query("SELECT 1 AS ok")
72
+ return TestConnectionResult(
73
+ success=True,
74
+ message="Connection successful.",
75
+ details={"row_count": result.row_count},
76
+ )
77
+ except Exception as e:
78
+ return TestConnectionResult(success=False, message=str(e))
79
+
80
+ async def query(self, sql: str, params: list | None = None) -> QueryResult:
81
+ import asyncio
82
+
83
+ client = await self._get_client()
84
+ start = time.monotonic()
85
+ try:
86
+ job = await asyncio.to_thread(client.query, sql)
87
+ rows_iter = await asyncio.to_thread(job.result)
88
+ rows = [dict(r.items()) for r in rows_iter]
89
+ fields = (
90
+ [
91
+ FieldInfo(name=s.name, data_type=s.field_type)
92
+ for s in rows_iter.schema
93
+ ]
94
+ if rows_iter.schema
95
+ else []
96
+ )
97
+ exec_time = int((time.monotonic() - start) * 1000)
98
+ return QueryResult(rows=rows, fields=fields, row_count=len(rows), execution_time=exec_time)
99
+ except Exception as e:
100
+ raise QueryError(str(e), sql) from e
101
+
102
+ async def list_schemas(self) -> list[str]:
103
+ import asyncio
104
+
105
+ client = await self._get_client()
106
+ datasets = await asyncio.to_thread(list, client.list_datasets())
107
+ return [d.dataset_id for d in datasets]
108
+
109
+ async def list_tables(self, schema: str | None = None) -> list[TableInfo]:
110
+ import asyncio
111
+
112
+ client = await self._get_client()
113
+ dataset_id = schema or self.config.dataset_id
114
+ if not dataset_id:
115
+ raise QueryError("dataset_id is required")
116
+ tables = await asyncio.to_thread(list, client.list_tables(dataset_id))
117
+ return [
118
+ TableInfo(
119
+ schema_name=dataset_id,
120
+ name=t.table_id,
121
+ type="table" if t.table_type == "TABLE" else "view",
122
+ )
123
+ for t in tables
124
+ ]
125
+
126
+ async def get_table_columns(self, schema: str, table: str) -> list[ColumnInfo]:
127
+ import asyncio
128
+
129
+ client = await self._get_client()
130
+ dataset_id = schema or self.config.dataset_id
131
+ ref = client.dataset(dataset_id).table(table)
132
+ tbl = await asyncio.to_thread(client.get_table, ref)
133
+ return [
134
+ ColumnInfo(
135
+ name=s.name,
136
+ data_type=s.field_type,
137
+ nullable=s.is_nullable,
138
+ )
139
+ for s in tbl.schema
140
+ ]