pgsemantic 0.2.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. pgsemantic-0.4.0/PKG-INFO +317 -0
  2. pgsemantic-0.4.0/README.md +280 -0
  3. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/__init__.py +1 -1
  4. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/cli.py +4 -0
  5. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/commands/apply.py +31 -6
  6. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/commands/index.py +102 -41
  7. pgsemantic-0.4.0/pgsemantic/commands/migrate.py +277 -0
  8. pgsemantic-0.4.0/pgsemantic/commands/retry.py +81 -0
  9. pgsemantic-0.4.0/pgsemantic/commands/search.py +235 -0
  10. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/commands/status.py +77 -4
  11. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/config.py +3 -0
  12. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/db/queue.py +93 -0
  13. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/db/vectors.py +221 -3
  14. pgsemantic-0.4.0/pgsemantic/embeddings/chunker.py +69 -0
  15. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/mcp_server/server.py +300 -12
  16. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/web/app.py +611 -56
  17. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/web/static/index.html +300 -198
  18. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/worker/daemon.py +102 -33
  19. pgsemantic-0.4.0/pgsemantic.egg-info/PKG-INFO +317 -0
  20. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic.egg-info/SOURCES.txt +3 -0
  21. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pyproject.toml +1 -1
  22. pgsemantic-0.2.0/PKG-INFO +0 -437
  23. pgsemantic-0.2.0/README.md +0 -400
  24. pgsemantic-0.2.0/pgsemantic/commands/search.py +0 -130
  25. pgsemantic-0.2.0/pgsemantic.egg-info/PKG-INFO +0 -437
  26. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/LICENSE +0 -0
  27. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/__main__.py +0 -0
  28. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/commands/__init__.py +0 -0
  29. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/commands/inspect.py +0 -0
  30. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/commands/integrate.py +0 -0
  31. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/commands/serve.py +0 -0
  32. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/commands/ui.py +0 -0
  33. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/commands/worker.py +0 -0
  34. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/db/__init__.py +0 -0
  35. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/db/client.py +0 -0
  36. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/db/introspect.py +0 -0
  37. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/embeddings/__init__.py +0 -0
  38. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/embeddings/base.py +0 -0
  39. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/embeddings/local.py +0 -0
  40. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/embeddings/ollama_provider.py +0 -0
  41. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/embeddings/openai_provider.py +0 -0
  42. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/exceptions.py +0 -0
  43. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/mcp_server/__init__.py +0 -0
  44. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/web/__init__.py +0 -0
  45. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic/worker/__init__.py +0 -0
  46. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic.egg-info/dependency_links.txt +0 -0
  47. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic.egg-info/entry_points.txt +0 -0
  48. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic.egg-info/requires.txt +0 -0
  49. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/pgsemantic.egg-info/top_level.txt +0 -0
  50. {pgsemantic-0.2.0 → pgsemantic-0.4.0}/setup.cfg +0 -0
@@ -0,0 +1,317 @@
1
+ Metadata-Version: 2.4
2
+ Name: pgsemantic
3
+ Version: 0.4.0
4
+ Summary: Zero-config semantic search bootstrap for any PostgreSQL database
5
+ Author: Sai Ram Varma Budharaju
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/varmabudharaju/pgsemantic
8
+ Project-URL: Issues, https://github.com/varmabudharaju/pgsemantic/issues
9
+ Keywords: pgvector,postgresql,semantic-search,rag,embeddings,mcp
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Requires-Python: >=3.10
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Requires-Dist: typer<1.0.0,>=0.9.0
19
+ Requires-Dist: rich<14.0.0,>=13.0.0
20
+ Requires-Dist: psycopg[binary]<4.0.0,>=3.1.0
21
+ Requires-Dist: pgvector<1.0.0,>=0.3.0
22
+ Requires-Dist: sentence-transformers<4.0.0,>=3.0.0
23
+ Requires-Dist: openai<2.0.0,>=1.0.0
24
+ Requires-Dist: mcp<2.0.0,>=1.0.0
25
+ Requires-Dist: httpx<1.0.0,>=0.27.0
26
+ Requires-Dist: python-dotenv<2.0.0,>=1.0.0
27
+ Requires-Dist: fastapi<1.0.0,>=0.110.0
28
+ Requires-Dist: uvicorn<1.0.0,>=0.27.0
29
+ Provides-Extra: dev
30
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
31
+ Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
32
+ Requires-Dist: pytest-benchmark>=4.0.0; extra == "dev"
33
+ Requires-Dist: ruff; extra == "dev"
34
+ Requires-Dist: mypy>=1.8.0; extra == "dev"
35
+ Requires-Dist: types-psycopg2; extra == "dev"
36
+ Dynamic: license-file
37
+
38
+ # pgsemantic
39
+
40
+ **Turn any PostgreSQL database into a semantic search engine. Set up in 60 seconds. No migrations, no new infrastructure.**
41
+
42
+ [![PyPI](https://img.shields.io/pypi/v/pgsemantic)](https://pypi.org/project/pgsemantic/)
43
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/downloads/)
44
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
45
+ [![Tests](https://img.shields.io/badge/tests-144%20passing-brightgreen)]()
46
+
47
+ pgsemantic connects to your existing PostgreSQL database, picks up your text columns, and makes them searchable by meaning. No data migration, no vector database, no pgvector expertise. It works on your live tables and keeps everything in sync automatically.
48
+
49
+ <img src="docs/screenshots/v4/06-visualize.png" width="100%" alt="Embedding visualization showing 3000 news articles clustered by topic — politics, covid, entertainment, sports"/>
50
+
51
+ *3,000 news articles visualized by semantic similarity. Each dot is a row in the database. Colors are auto-detected topic clusters. This is what your data looks like when you can search it by meaning.*
52
+
53
+ ---
54
+
55
+ ## What makes this different
56
+
57
+ Most vector search tools ask you to set up a separate vector database, write ETL pipelines, and maintain sync logic. pgsemantic works directly on your existing PostgreSQL tables:
58
+
59
+ - **No new database** — embeddings live alongside your data (inline column or shadow table)
60
+ - **No ETL** — a database trigger keeps embeddings in sync as rows change
61
+ - **No expertise needed** — point it at a table, pick a column, done
62
+ - **Free by default** — runs a local embedding model on your machine (no API keys)
63
+
64
+ ---
65
+
66
+ ## See it in action
67
+
68
+ ### Connect to any PostgreSQL database
69
+
70
+ <img src="docs/screenshots/v4/01-connection.png" width="100%" alt="Connection page showing connected state with pgvector 0.8.2"/>
71
+
72
+ *Paste your connection string, test it, done. Works with localhost, Supabase, Neon, RDS — anything with pgvector.*
73
+
74
+ ### Browse and explore your tables
75
+
76
+ <img src="docs/screenshots/v4/02-tables.png" width="100%" alt="Table browser showing clinical_trials, news_articles, and products with column info and row counts"/>
77
+
78
+ *See every table, column types, and row counts. Click Inspect to score columns for semantic search suitability.*
79
+
80
+ ### Set up with chunking for long documents
81
+
82
+ <img src="docs/screenshots/v4/03-apply-chunked.png" width="100%" alt="Apply page with column picker, model selection, and chunking enabled"/>
83
+
84
+ *Pick columns, choose a model, enable chunking for long text. Chunking splits documents into overlapping paragraphs so search finds the best paragraph, not just the best document.*
85
+
86
+ ### Search by meaning, not keywords
87
+
88
+ <img src="docs/screenshots/v4/05-search.png" width="100%" alt="Search results for 'economic recession' showing articles about inflation, unemployment, and consumer prices"/>
89
+
90
+ *Query: "economic recession" — returns articles about inflation, unemployment, consumer prices. These results share the concept, not the exact words.*
91
+
92
+ ### Search across all your tables at once
93
+
94
+ <img src="docs/screenshots/v4/10-cross-table-search.png" width="100%" alt="Cross-table search results showing clinical trials and news articles mixed together with table badges"/>
95
+
96
+ *"health treatment side effects" across clinical_trials and news_articles. Results ranked by similarity regardless of source table. Purple badges show where each result came from.*
97
+
98
+ ### See how your data clusters
99
+
100
+ <img src="docs/screenshots/v4/06-visualize.png" width="100%" alt="2D scatter plot of 3000 embeddings with colored clusters labeled by topic"/>
101
+
102
+ *3,000 embeddings reduced to 2D with PCA. K-means clustering auto-labels each group with its top keywords. Click any point to see the full row.*
103
+
104
+ <img src="docs/screenshots/v4/07-visualize-detail.png" width="100%" alt="Row detail drawer showing full article data from a clicked point in the scatter plot"/>
105
+
106
+ ### Run SQL queries directly
107
+
108
+ <img src="docs/screenshots/v4/08-sql-query.png" width="100%" alt="SQL Query page showing category counts from news_articles table"/>
109
+
110
+ *Built-in SQL query tool. Read-only, 10-second timeout, 100-row limit. Safe to explore.*
111
+
112
+ ### Monitor everything from one dashboard
113
+
114
+ <img src="docs/screenshots/v4/04-dashboard.png" width="100%" alt="Dashboard showing 2 tables, 8003 embeddings, 100% coverage, 0 pending"/>
115
+
116
+ *Coverage, pending queue, worker health — all in one view.*
117
+
118
+ ### 10 MCP tools for AI agents
119
+
120
+ <img src="docs/screenshots/v4/09-mcp-tools.png" width="100%" alt="MCP integration page showing 10 available tools including semantic search, SQL execution, and schema context"/>
121
+
122
+ *Connect Claude Desktop, Cursor, or any MCP-compatible AI agent. They can search semantically, run SQL queries, and explore your schema.*
123
+
124
+ ---
125
+
126
+ ## Quick start
127
+
128
+ ```bash
129
+ pip install pgsemantic
130
+
131
+ # Launch the web UI
132
+ pgsemantic ui
133
+
134
+ # Or use the CLI
135
+ pgsemantic inspect # Scan your DB, score columns
136
+ pgsemantic apply --table articles --column body # Set up semantic search
137
+ pgsemantic index --table articles # Embed existing rows
138
+ pgsemantic search "climate policy effects" # Search by meaning
139
+ ```
140
+
141
+ Open [http://localhost:8080](http://localhost:8080) and walk through setup visually:
142
+
143
+ 1. **Connection** — paste your PostgreSQL URL
144
+ 2. **Tables** — browse your database
145
+ 3. **Apply** — pick columns, choose a model, one click
146
+ 4. **Index** — embed all rows (progress bar, resumable)
147
+ 5. **Search** — natural language queries, instantly
148
+
149
+ ---
150
+
151
+ ## Features
152
+
153
+ ### Core
154
+
155
+ | Feature | Description |
156
+ |---------|-------------|
157
+ | **Semantic search** | Search by meaning, not keywords. "budget travel" finds "affordable hostels" |
158
+ | **Cross-table search** | Search all configured tables at once — results merged by similarity |
159
+ | **Auto-chunking** | Split long documents into overlapping chunks for precise paragraph-level search |
160
+ | **Hybrid search** | Combine semantic similarity with SQL WHERE filters in one query |
161
+ | **Multi-column embedding** | Embed title + description + tags as one combined vector |
162
+ | **Automatic sync** | Database triggers keep embeddings fresh on every INSERT/UPDATE/DELETE |
163
+
164
+ ### Operations
165
+
166
+ | Feature | Description |
167
+ |---------|-------------|
168
+ | **Zero-downtime model migration** | Switch embedding models without breaking search — atomic swap |
169
+ | **Pipelined indexing** | Embeds batch N+1 while writing batch N — ~2x throughput |
170
+ | **Background worker** | Daemon process for continuous embedding sync with retry logic |
171
+ | **Shadow table storage** | Store embeddings separately without modifying your source table |
172
+
173
+ ### Visualization & Analytics
174
+
175
+ | Feature | Description |
176
+ |---------|-------------|
177
+ | **Embedding scatter plot** | 2D PCA visualization with auto-detected clusters and keyword labels |
178
+ | **SQL query tool** | Run read-only queries from the UI (10s timeout, 100-row limit) |
179
+ | **Status dashboard** | Coverage %, pending queue, failed jobs, worker health — all in one view |
180
+
181
+ ### AI Agent Integration (MCP)
182
+
183
+ 10 tools for Claude Desktop, Cursor, and any MCP-compatible agent:
184
+
185
+ | Tool | What it does |
186
+ |------|-------------|
187
+ | `semantic_search` | Find rows by meaning |
188
+ | `hybrid_search` | Semantic + SQL filters |
189
+ | `search_all_tables` | Cross-table search |
190
+ | `get_schema_context` | Table schemas for SQL generation |
191
+ | `execute_safe_sql` | Run read-only SQL queries safely |
192
+ | `get_embedding_status` | Check embedding coverage |
193
+ | `list_tables` | Browse all tables |
194
+ | `get_sample_rows` | Preview table data |
195
+ | `inspect_columns` | Score columns for search suitability |
196
+ | `list_configured_tables` | List configured tables |
197
+
198
+ ---
199
+
200
+ ## Embedding models
201
+
202
+ | Model | Dimensions | Cost | Best for |
203
+ |-------|-----------|------|---------|
204
+ | **Local Fast** (all-MiniLM-L6-v2) | 384 | Free | Getting started, most use cases |
205
+ | **Local Better** (all-mpnet-base-v2) | 768 | Free | Higher accuracy |
206
+ | **OpenAI Small** (text-embedding-3-small) | 1536 | ~$0.02/1M tokens | Production, multilingual |
207
+ | **OpenAI Large** (text-embedding-3-large) | 3072 | ~$0.13/1M tokens | Maximum accuracy |
208
+ | **Ollama** (nomic-embed-text) | 768 | Free | Self-hosted, privacy-sensitive |
209
+
210
+ Switch models anytime with zero-downtime migration:
211
+
212
+ ```bash
213
+ pgsemantic migrate --table articles --model openai
214
+ ```
215
+
216
+ ---
217
+
218
+ ## Works with any PostgreSQL
219
+
220
+ - Local PostgreSQL + pgvector
221
+ - **Supabase** (enable pgvector in Dashboard > Database > Extensions)
222
+ - **Neon**
223
+ - **Amazon RDS** / Aurora PostgreSQL
224
+ - **Google Cloud SQL** for PostgreSQL
225
+ - Any self-hosted PostgreSQL with pgvector
226
+
227
+ ---
228
+
229
+ ## Architecture
230
+
231
+ ```
232
+ Your App ──────────────────────────────────────────────────┐
233
+
234
+ CLI (Typer) ─┐ │
235
+ Web UI (FastAPI) ─┤──> Embedding Providers ──> PostgreSQL + pgvector
236
+ MCP Server (FastMCP) ─┘ (local/openai/ollama) │
237
+ ├── Source tables (your data)
238
+ Background Worker ──> Job Queue ────────────────────├── Shadow tables (embeddings)
239
+ └── HNSW indexes (fast search)
240
+ ```
241
+
242
+ - **No external services** — everything runs locally (unless you choose OpenAI)
243
+ - **No build step** — the web UI is vanilla HTML/JS/CSS in a single file
244
+ - **No external dependencies in the frontend** — no React, no Tailwind, no CDN
245
+
246
+ ---
247
+
248
+ ## Security
249
+
250
+ - Database URL **never sent to the browser** — stays server-side only
251
+ - **CSRF protection** on all POST endpoints
252
+ - **SQL injection prevention** via Pydantic validation + regex on identifiers
253
+ - **Rate limiting** — 120 requests/minute per IP
254
+ - **Content Security Policy**, X-Frame-Options, and security headers on all responses
255
+ - SQL Query tool enforces **read-only transactions** with **10-second timeout**
256
+ - `.env` written with **chmod 600** — owner-read-only
257
+
258
+ ---
259
+
260
+ ## CLI reference
261
+
262
+ ```bash
263
+ pgsemantic inspect # Scan DB, score columns
264
+ pgsemantic apply # Set up semantic search on a table
265
+ pgsemantic index # Bulk embed existing rows
266
+ pgsemantic search "query" # Search (omit --table to search all)
267
+ pgsemantic migrate # Switch embedding models (zero downtime)
268
+ pgsemantic worker # Background embedding sync daemon
269
+ pgsemantic serve # MCP server for AI agents
270
+ pgsemantic status # Health dashboard
271
+ pgsemantic ui # Web dashboard
272
+ pgsemantic retry # Reset failed embedding jobs
273
+ ```
274
+
275
+ ---
276
+
277
+ ## Development
278
+
279
+ ```bash
280
+ git clone https://github.com/varmabudharaju/pgvector-setup.git
281
+ cd pgvector-setup
282
+ pip install -e ".[dev]"
283
+ docker-compose up -d # Postgres + pgvector
284
+ pytest tests/unit/ -v # 144 tests, all passing
285
+ ruff check . # Lint
286
+ ```
287
+
288
+ ---
289
+
290
+ ## FAQ
291
+
292
+ **Does it work with tables that already have data?**
293
+ Yes. Run `index` to embed existing rows. New rows are handled automatically by the trigger.
294
+
295
+ **Which embedding model should I pick?**
296
+ Start with Local Fast — it's free and works offline. Switch to OpenAI for multilingual or higher accuracy.
297
+
298
+ **Can I search across multiple tables?**
299
+ Yes. `pgsemantic search "query"` (without `--table`) searches all configured tables at once.
300
+
301
+ **Can I change the model after indexing?**
302
+ Yes. `pgsemantic migrate --table T --model openai` re-embeds in the background with zero downtime.
303
+
304
+ **What about long documents?**
305
+ Use `--chunked` when applying. This splits text into overlapping chunks so search finds the best paragraph, not just the best document.
306
+
307
+ **Does it slow down my database?**
308
+ The trigger writes a small job record to a queue table. The actual embedding happens asynchronously in the background worker, not in your transaction.
309
+
310
+ **How do I connect AI agents?**
311
+ Add the MCP config snippet (shown on the MCP page in the UI) to your Claude Desktop or Cursor config. Agents get 10 tools including semantic search, SQL queries, and schema exploration.
312
+
313
+ ---
314
+
315
+ ## License
316
+
317
+ MIT
@@ -0,0 +1,280 @@
1
+ # pgsemantic
2
+
3
+ **Turn any PostgreSQL database into a semantic search engine. Set up in 60 seconds. No migrations, no new infrastructure.**
4
+
5
+ [![PyPI](https://img.shields.io/pypi/v/pgsemantic)](https://pypi.org/project/pgsemantic/)
6
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/downloads/)
7
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
8
+ [![Tests](https://img.shields.io/badge/tests-144%20passing-brightgreen)]()
9
+
10
+ pgsemantic connects to your existing PostgreSQL database, picks up your text columns, and makes them searchable by meaning. No data migration, no vector database, no pgvector expertise. It works on your live tables and keeps everything in sync automatically.
11
+
12
+ <img src="docs/screenshots/v4/06-visualize.png" width="100%" alt="Embedding visualization showing 3000 news articles clustered by topic — politics, covid, entertainment, sports"/>
13
+
14
+ *3,000 news articles visualized by semantic similarity. Each dot is a row in the database. Colors are auto-detected topic clusters. This is what your data looks like when you can search it by meaning.*
15
+
16
+ ---
17
+
18
+ ## What makes this different
19
+
20
+ Most vector search tools ask you to set up a separate vector database, write ETL pipelines, and maintain sync logic. pgsemantic works directly on your existing PostgreSQL tables:
21
+
22
+ - **No new database** — embeddings live alongside your data (inline column or shadow table)
23
+ - **No ETL** — a database trigger keeps embeddings in sync as rows change
24
+ - **No expertise needed** — point it at a table, pick a column, done
25
+ - **Free by default** — runs a local embedding model on your machine (no API keys)
26
+
27
+ ---
28
+
29
+ ## See it in action
30
+
31
+ ### Connect to any PostgreSQL database
32
+
33
+ <img src="docs/screenshots/v4/01-connection.png" width="100%" alt="Connection page showing connected state with pgvector 0.8.2"/>
34
+
35
+ *Paste your connection string, test it, done. Works with localhost, Supabase, Neon, RDS — anything with pgvector.*
36
+
37
+ ### Browse and explore your tables
38
+
39
+ <img src="docs/screenshots/v4/02-tables.png" width="100%" alt="Table browser showing clinical_trials, news_articles, and products with column info and row counts"/>
40
+
41
+ *See every table, column types, and row counts. Click Inspect to score columns for semantic search suitability.*
42
+
43
+ ### Set up with chunking for long documents
44
+
45
+ <img src="docs/screenshots/v4/03-apply-chunked.png" width="100%" alt="Apply page with column picker, model selection, and chunking enabled"/>
46
+
47
+ *Pick columns, choose a model, enable chunking for long text. Chunking splits documents into overlapping paragraphs so search finds the best paragraph, not just the best document.*
48
+
49
+ ### Search by meaning, not keywords
50
+
51
+ <img src="docs/screenshots/v4/05-search.png" width="100%" alt="Search results for 'economic recession' showing articles about inflation, unemployment, and consumer prices"/>
52
+
53
+ *Query: "economic recession" — returns articles about inflation, unemployment, consumer prices. These results share the concept, not the exact words.*
54
+
55
+ ### Search across all your tables at once
56
+
57
+ <img src="docs/screenshots/v4/10-cross-table-search.png" width="100%" alt="Cross-table search results showing clinical trials and news articles mixed together with table badges"/>
58
+
59
+ *"health treatment side effects" across clinical_trials and news_articles. Results ranked by similarity regardless of source table. Purple badges show where each result came from.*
60
+
61
+ ### See how your data clusters
62
+
63
+ <img src="docs/screenshots/v4/06-visualize.png" width="100%" alt="2D scatter plot of 3000 embeddings with colored clusters labeled by topic"/>
64
+
65
+ *3,000 embeddings reduced to 2D with PCA. K-means clustering auto-labels each group with its top keywords. Click any point to see the full row.*
66
+
67
+ <img src="docs/screenshots/v4/07-visualize-detail.png" width="100%" alt="Row detail drawer showing full article data from a clicked point in the scatter plot"/>
68
+
69
+ ### Run SQL queries directly
70
+
71
+ <img src="docs/screenshots/v4/08-sql-query.png" width="100%" alt="SQL Query page showing category counts from news_articles table"/>
72
+
73
+ *Built-in SQL query tool. Read-only, 10-second timeout, 100-row limit. Safe to explore.*
74
+
75
+ ### Monitor everything from one dashboard
76
+
77
+ <img src="docs/screenshots/v4/04-dashboard.png" width="100%" alt="Dashboard showing 2 tables, 8003 embeddings, 100% coverage, 0 pending"/>
78
+
79
+ *Coverage, pending queue, worker health — all in one view.*
80
+
81
+ ### 10 MCP tools for AI agents
82
+
83
+ <img src="docs/screenshots/v4/09-mcp-tools.png" width="100%" alt="MCP integration page showing 10 available tools including semantic search, SQL execution, and schema context"/>
84
+
85
+ *Connect Claude Desktop, Cursor, or any MCP-compatible AI agent. They can search semantically, run SQL queries, and explore your schema.*
86
+
87
+ ---
88
+
89
+ ## Quick start
90
+
91
+ ```bash
92
+ pip install pgsemantic
93
+
94
+ # Launch the web UI
95
+ pgsemantic ui
96
+
97
+ # Or use the CLI
98
+ pgsemantic inspect # Scan your DB, score columns
99
+ pgsemantic apply --table articles --column body # Set up semantic search
100
+ pgsemantic index --table articles # Embed existing rows
101
+ pgsemantic search "climate policy effects" # Search by meaning
102
+ ```
103
+
104
+ Open [http://localhost:8080](http://localhost:8080) and walk through setup visually:
105
+
106
+ 1. **Connection** — paste your PostgreSQL URL
107
+ 2. **Tables** — browse your database
108
+ 3. **Apply** — pick columns, choose a model, one click
109
+ 4. **Index** — embed all rows (progress bar, resumable)
110
+ 5. **Search** — natural language queries, instantly
111
+
112
+ ---
113
+
114
+ ## Features
115
+
116
+ ### Core
117
+
118
+ | Feature | Description |
119
+ |---------|-------------|
120
+ | **Semantic search** | Search by meaning, not keywords. "budget travel" finds "affordable hostels" |
121
+ | **Cross-table search** | Search all configured tables at once — results merged by similarity |
122
+ | **Auto-chunking** | Split long documents into overlapping chunks for precise paragraph-level search |
123
+ | **Hybrid search** | Combine semantic similarity with SQL WHERE filters in one query |
124
+ | **Multi-column embedding** | Embed title + description + tags as one combined vector |
125
+ | **Automatic sync** | Database triggers keep embeddings fresh on every INSERT/UPDATE/DELETE |
126
+
127
+ ### Operations
128
+
129
+ | Feature | Description |
130
+ |---------|-------------|
131
+ | **Zero-downtime model migration** | Switch embedding models without breaking search — atomic swap |
132
+ | **Pipelined indexing** | Embeds batch N+1 while writing batch N — ~2x throughput |
133
+ | **Background worker** | Daemon process for continuous embedding sync with retry logic |
134
+ | **Shadow table storage** | Store embeddings separately without modifying your source table |
135
+
136
+ ### Visualization & Analytics
137
+
138
+ | Feature | Description |
139
+ |---------|-------------|
140
+ | **Embedding scatter plot** | 2D PCA visualization with auto-detected clusters and keyword labels |
141
+ | **SQL query tool** | Run read-only queries from the UI (10s timeout, 100-row limit) |
142
+ | **Status dashboard** | Coverage %, pending queue, failed jobs, worker health — all in one view |
143
+
144
+ ### AI Agent Integration (MCP)
145
+
146
+ 10 tools for Claude Desktop, Cursor, and any MCP-compatible agent:
147
+
148
+ | Tool | What it does |
149
+ |------|-------------|
150
+ | `semantic_search` | Find rows by meaning |
151
+ | `hybrid_search` | Semantic + SQL filters |
152
+ | `search_all_tables` | Cross-table search |
153
+ | `get_schema_context` | Table schemas for SQL generation |
154
+ | `execute_safe_sql` | Run read-only SQL queries safely |
155
+ | `get_embedding_status` | Check embedding coverage |
156
+ | `list_tables` | Browse all tables |
157
+ | `get_sample_rows` | Preview table data |
158
+ | `inspect_columns` | Score columns for search suitability |
159
+ | `list_configured_tables` | List configured tables |
160
+
161
+ ---
162
+
163
+ ## Embedding models
164
+
165
+ | Model | Dimensions | Cost | Best for |
166
+ |-------|-----------|------|---------|
167
+ | **Local Fast** (all-MiniLM-L6-v2) | 384 | Free | Getting started, most use cases |
168
+ | **Local Better** (all-mpnet-base-v2) | 768 | Free | Higher accuracy |
169
+ | **OpenAI Small** (text-embedding-3-small) | 1536 | ~$0.02/1M tokens | Production, multilingual |
170
+ | **OpenAI Large** (text-embedding-3-large) | 3072 | ~$0.13/1M tokens | Maximum accuracy |
171
+ | **Ollama** (nomic-embed-text) | 768 | Free | Self-hosted, privacy-sensitive |
172
+
173
+ Switch models anytime with zero-downtime migration:
174
+
175
+ ```bash
176
+ pgsemantic migrate --table articles --model openai
177
+ ```
178
+
179
+ ---
180
+
181
+ ## Works with any PostgreSQL
182
+
183
+ - Local PostgreSQL + pgvector
184
+ - **Supabase** (enable pgvector in Dashboard > Database > Extensions)
185
+ - **Neon**
186
+ - **Amazon RDS** / Aurora PostgreSQL
187
+ - **Google Cloud SQL** for PostgreSQL
188
+ - Any self-hosted PostgreSQL with pgvector
189
+
190
+ ---
191
+
192
+ ## Architecture
193
+
194
+ ```
195
+ Your App ──────────────────────────────────────────────────┐
196
+
197
+ CLI (Typer) ─┐ │
198
+ Web UI (FastAPI) ─┤──> Embedding Providers ──> PostgreSQL + pgvector
199
+ MCP Server (FastMCP) ─┘ (local/openai/ollama) │
200
+ ├── Source tables (your data)
201
+ Background Worker ──> Job Queue ────────────────────├── Shadow tables (embeddings)
202
+ └── HNSW indexes (fast search)
203
+ ```
204
+
205
+ - **No external services** — everything runs locally (unless you choose OpenAI)
206
+ - **No build step** — the web UI is vanilla HTML/JS/CSS in a single file
207
+ - **No external dependencies in the frontend** — no React, no Tailwind, no CDN
208
+
209
+ ---
210
+
211
+ ## Security
212
+
213
+ - Database URL **never sent to the browser** — stays server-side only
214
+ - **CSRF protection** on all POST endpoints
215
+ - **SQL injection prevention** via Pydantic validation + regex on identifiers
216
+ - **Rate limiting** — 120 requests/minute per IP
217
+ - **Content Security Policy**, X-Frame-Options, and security headers on all responses
218
+ - SQL Query tool enforces **read-only transactions** with **10-second timeout**
219
+ - `.env` written with **chmod 600** — owner-read-only
220
+
221
+ ---
222
+
223
+ ## CLI reference
224
+
225
+ ```bash
226
+ pgsemantic inspect # Scan DB, score columns
227
+ pgsemantic apply # Set up semantic search on a table
228
+ pgsemantic index # Bulk embed existing rows
229
+ pgsemantic search "query" # Search (omit --table to search all)
230
+ pgsemantic migrate # Switch embedding models (zero downtime)
231
+ pgsemantic worker # Background embedding sync daemon
232
+ pgsemantic serve # MCP server for AI agents
233
+ pgsemantic status # Health dashboard
234
+ pgsemantic ui # Web dashboard
235
+ pgsemantic retry # Reset failed embedding jobs
236
+ ```
237
+
238
+ ---
239
+
240
+ ## Development
241
+
242
+ ```bash
243
+ git clone https://github.com/varmabudharaju/pgvector-setup.git
244
+ cd pgvector-setup
245
+ pip install -e ".[dev]"
246
+ docker-compose up -d # Postgres + pgvector
247
+ pytest tests/unit/ -v # 144 tests, all passing
248
+ ruff check . # Lint
249
+ ```
250
+
251
+ ---
252
+
253
+ ## FAQ
254
+
255
+ **Does it work with tables that already have data?**
256
+ Yes. Run `index` to embed existing rows. New rows are handled automatically by the trigger.
257
+
258
+ **Which embedding model should I pick?**
259
+ Start with Local Fast — it's free and works offline. Switch to OpenAI for multilingual or higher accuracy.
260
+
261
+ **Can I search across multiple tables?**
262
+ Yes. `pgsemantic search "query"` (without `--table`) searches all configured tables at once.
263
+
264
+ **Can I change the model after indexing?**
265
+ Yes. `pgsemantic migrate --table T --model openai` re-embeds in the background with zero downtime.
266
+
267
+ **What about long documents?**
268
+ Use `--chunked` when applying. This splits text into overlapping chunks so search finds the best paragraph, not just the best document.
269
+
270
+ **Does it slow down my database?**
271
+ The trigger writes a small job record to a queue table. The actual embedding happens asynchronously in the background worker, not in your transaction.
272
+
273
+ **How do I connect AI agents?**
274
+ Add the MCP config snippet (shown on the MCP page in the UI) to your Claude Desktop or Cursor config. Agents get 10 tools including semantic search, SQL queries, and schema exploration.
275
+
276
+ ---
277
+
278
+ ## License
279
+
280
+ MIT
@@ -1,3 +1,3 @@
1
1
  """pgsemantic: Zero-config semantic search bootstrap for PostgreSQL."""
2
2
 
3
- __version__ = "0.2.0"
3
+ __version__ = "0.4.0"
@@ -37,6 +37,8 @@ from pgsemantic.commands.apply import apply_command # noqa: E402
37
37
  from pgsemantic.commands.index import index_command # noqa: E402
38
38
  from pgsemantic.commands.inspect import inspect_command # noqa: E402
39
39
  from pgsemantic.commands.integrate import integrate_command # noqa: E402
40
+ from pgsemantic.commands.migrate import migrate_command # noqa: E402
41
+ from pgsemantic.commands.retry import retry_command # noqa: E402
40
42
  from pgsemantic.commands.search import search_command # noqa: E402
41
43
  from pgsemantic.commands.serve import serve_command # noqa: E402
42
44
  from pgsemantic.commands.status import status_command # noqa: E402
@@ -51,4 +53,6 @@ app.command(name="worker")(worker_command)
51
53
  app.command(name="serve")(serve_command)
52
54
  app.command(name="status")(status_command)
53
55
  app.command(name="integrate")(integrate_command)
56
+ app.command(name="migrate")(migrate_command)
54
57
  app.command(name="ui")(ui_command)
58
+ app.command(name="retry")(retry_command)