embedflow 0.1.1__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. {embedflow-0.1.1 → embedflow-0.2.0}/CHANGELOG.md +8 -0
  2. {embedflow-0.1.1 → embedflow-0.2.0}/CITATION.cff +1 -1
  3. {embedflow-0.1.1 → embedflow-0.2.0}/MANIFEST.in +2 -1
  4. {embedflow-0.1.1 → embedflow-0.2.0}/PKG-INFO +22 -11
  5. {embedflow-0.1.1 → embedflow-0.2.0}/README.md +5 -3
  6. {embedflow-0.1.1 → embedflow-0.2.0}/README_PYPI.md +13 -5
  7. {embedflow-0.1.1 → embedflow-0.2.0}/docs/api.md +6 -2
  8. {embedflow-0.1.1 → embedflow-0.2.0}/docs/cli.md +1 -1
  9. {embedflow-0.1.1 → embedflow-0.2.0}/docs/configuration.md +11 -1
  10. {embedflow-0.1.1 → embedflow-0.2.0}/docs/economics.md +1 -1
  11. {embedflow-0.1.1 → embedflow-0.2.0}/docs/installation.md +7 -0
  12. embedflow-0.2.0/docs/integrations/pgvector.md +140 -0
  13. {embedflow-0.1.1 → embedflow-0.2.0}/docs/limitations.md +1 -1
  14. {embedflow-0.1.1 → embedflow-0.2.0}/docs/releasing.md +8 -8
  15. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/__init__.py +2 -2
  16. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/analysis.py +5 -1
  17. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/cli.py +117 -30
  18. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/config.py +40 -5
  19. embedflow-0.2.0/embedflow/indexes/__init__.py +9 -0
  20. embedflow-0.2.0/embedflow/indexes/pgvector_backend.py +685 -0
  21. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/facade.py +61 -12
  22. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/materializer.py +18 -6
  23. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/runtime.py +41 -6
  24. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/api.py +13 -4
  25. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/engine.py +8 -2
  26. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/PKG-INFO +22 -11
  27. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/SOURCES.txt +10 -0
  28. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/requires.txt +8 -4
  29. embedflow-0.2.0/examples/pgvector/README.md +28 -0
  30. embedflow-0.2.0/examples/pgvector/build_index.py +57 -0
  31. embedflow-0.2.0/examples/pgvector/compose.yaml +20 -0
  32. embedflow-0.2.0/examples/pgvector/embedflow.yaml +48 -0
  33. embedflow-0.2.0/examples/pgvector/init.sql +9 -0
  34. embedflow-0.2.0/examples/pgvector/queries.jsonl +4 -0
  35. embedflow-0.2.0/examples/pgvector/run_demo.sh +11 -0
  36. {embedflow-0.1.1 → embedflow-0.2.0}/pyproject.toml +7 -5
  37. {embedflow-0.1.1 → embedflow-0.2.0}/scripts/release_gate.py +19 -2
  38. embedflow-0.2.0/scripts/validate_pgvector_10k.py +812 -0
  39. embedflow-0.1.1/embedflow/indexes/__init__.py +0 -5
  40. {embedflow-0.1.1 → embedflow-0.2.0}/CONTRIBUTING.md +0 -0
  41. {embedflow-0.1.1 → embedflow-0.2.0}/LICENSE +0 -0
  42. {embedflow-0.1.1 → embedflow-0.2.0}/SECURITY.md +0 -0
  43. {embedflow-0.1.1 → embedflow-0.2.0}/docs/assets/README.md +0 -0
  44. {embedflow-0.1.1 → embedflow-0.2.0}/docs/assets/candidate-gap-example.svg +0 -0
  45. {embedflow-0.1.1 → embedflow-0.2.0}/docs/assets/dashboard-screenshot.md +0 -0
  46. {embedflow-0.1.1 → embedflow-0.2.0}/docs/assets/terminal-demo.txt +0 -0
  47. {embedflow-0.1.1 → embedflow-0.2.0}/docs/concepts.md +0 -0
  48. {embedflow-0.1.1 → embedflow-0.2.0}/docs/contributing-benchmarks.md +0 -0
  49. {embedflow-0.1.1 → embedflow-0.2.0}/docs/integrations/faiss.md +0 -0
  50. {embedflow-0.1.1 → embedflow-0.2.0}/docs/integrations/qdrant.md +0 -0
  51. {embedflow-0.1.1 → embedflow-0.2.0}/docs/methodology.md +0 -0
  52. {embedflow-0.1.1 → embedflow-0.2.0}/docs/quickstart.md +0 -0
  53. {embedflow-0.1.1 → embedflow-0.2.0}/docs/registry.md +0 -0
  54. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/__main__.py +0 -0
  55. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/cache/__init__.py +0 -0
  56. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/cache/base.py +0 -0
  57. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/cache/persistent_cache.py +0 -0
  58. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/__init__.py +0 -0
  59. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/candidate_gap.py +0 -0
  60. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/containment.py +0 -0
  61. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/evaluate.py +0 -0
  62. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/metrics.py +0 -0
  63. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/migration_depth.py +0 -0
  64. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/probe.py +0 -0
  65. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/report.py +0 -0
  66. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/t2.py +0 -0
  67. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/__init__.py +0 -0
  68. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/__init__.py +0 -0
  69. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/benchmark_profiles.jsonl +0 -0
  70. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/checksums.sha256 +0 -0
  71. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/migrations.jsonl +0 -0
  72. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/registry_manifest.json +0 -0
  73. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/research_summaries.json +0 -0
  74. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/schema_version.json +0 -0
  75. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/frozen/T2_V1_FROZEN_SPEC.md +0 -0
  76. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/frozen/T2_V1_FROZEN_SPEC.sha256 +0 -0
  77. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/indexes/base.py +0 -0
  78. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/indexes/faiss_backend.py +0 -0
  79. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/indexes/qdrant_backend.py +0 -0
  80. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/metrics/__init__.py +0 -0
  81. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/metrics/latency.py +0 -0
  82. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/__init__.py +0 -0
  83. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/compatibility.py +0 -0
  84. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/planner.py +0 -0
  85. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/state.py +0 -0
  86. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/models/__init__.py +0 -0
  87. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/models/base.py +0 -0
  88. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/models/huggingface.py +0 -0
  89. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/registry/__init__.py +0 -0
  90. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/registry/loader.py +0 -0
  91. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/registry/matcher.py +0 -0
  92. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/registry/schema.py +0 -0
  93. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/__init__.py +0 -0
  94. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/factory.py +0 -0
  95. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/schemas.py +0 -0
  96. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/dependency_links.txt +0 -0
  97. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/entry_points.txt +0 -0
  98. {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/top_level.txt +0 -0
  99. {embedflow-0.1.1 → embedflow-0.2.0}/examples/faiss/README.md +0 -0
  100. {embedflow-0.1.1 → embedflow-0.2.0}/examples/faiss/documents.jsonl +0 -0
  101. {embedflow-0.1.1 → embedflow-0.2.0}/examples/faiss/embedflow.yaml +0 -0
  102. {embedflow-0.1.1 → embedflow-0.2.0}/examples/faiss/queries.jsonl +0 -0
  103. {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/README.md +0 -0
  104. {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/build_index.py +0 -0
  105. {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/documents.jsonl +0 -0
  106. {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/embedflow.yaml +0 -0
  107. {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/queries.jsonl +0 -0
  108. {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/README.md +0 -0
  109. {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/documents.jsonl +0 -0
  110. {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/embedflow.yaml +0 -0
  111. {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/qrels.json +0 -0
  112. {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/queries.jsonl +0 -0
  113. {embedflow-0.1.1 → embedflow-0.2.0}/frozen/T2_V1_FROZEN_SPEC.md +0 -0
  114. {embedflow-0.1.1 → embedflow-0.2.0}/frozen/T2_V1_FROZEN_SPEC.sha256 +0 -0
  115. {embedflow-0.1.1 → embedflow-0.2.0}/requirements-dev.txt +0 -0
  116. {embedflow-0.1.1 → embedflow-0.2.0}/requirements.txt +0 -0
  117. {embedflow-0.1.1 → embedflow-0.2.0}/scripts/real_qdrant_smoke.py +0 -0
  118. {embedflow-0.1.1 → embedflow-0.2.0}/scripts/run_demo.sh +0 -0
  119. {embedflow-0.1.1 → embedflow-0.2.0}/scripts/run_tests.sh +0 -0
  120. {embedflow-0.1.1 → embedflow-0.2.0}/setup.cfg +0 -0
  121. {embedflow-0.1.1 → embedflow-0.2.0}/src/__init__.py +0 -0
  122. {embedflow-0.1.1 → embedflow-0.2.0}/src/embed.py +0 -0
  123. {embedflow-0.1.1 → embedflow-0.2.0}/src/probe_features.py +0 -0
  124. {embedflow-0.1.1 → embedflow-0.2.0}/src/storage.py +0 -0
  125. {embedflow-0.1.1 → embedflow-0.2.0}/src/t2_v1.py +0 -0
  126. {embedflow-0.1.1 → embedflow-0.2.0}/src/utils.py +0 -0
@@ -1,5 +1,13 @@
1
1
  # Changelog
2
2
 
3
+ ## v0.2.0 — pgvector backend
4
+
5
+ - Added a read-only pgvector backend for existing PostgreSQL vector tables.
6
+ - Added cosine, Euclidean/L2, and inner-product retrieval with safe SQL
7
+ identifier composition.
8
+ - Added pgvector table auditing, optional transaction-local HNSW/IVFFlat
9
+ settings, Docker example, and numerical/SQL-safety tests.
10
+
3
11
  ## v0.1.0 — initial public release
4
12
 
5
13
  - Candidate-compatibility analysis and Mode A evaluation workflows.
@@ -2,7 +2,7 @@ cff-version: 1.2.0
2
2
  title: "EmbedFlow: Upgrading Legacy Embeddings Without Full Upfront Re-Embedding"
3
3
  message: "If EmbedFlow contributes to your work, please cite this software release."
4
4
  type: software
5
- version: 0.1.1
5
+ version: 0.2.0
6
6
  date-released: 2026-09-06
7
7
  repository-code: "https://github.com/arnsri33/embedflow"
8
8
  url: "https://github.com/arnsri33/embedflow"
@@ -1,10 +1,11 @@
1
1
  include README.md README_PYPI.md LICENSE CHANGELOG.md CONTRIBUTING.md SECURITY.md CITATION.cff requirements.txt requirements-dev.txt
2
2
  recursive-include docs *
3
- recursive-include examples *.md *.yaml *.json *.jsonl *.py
3
+ recursive-include examples *.md *.yaml *.json *.jsonl *.py *.sql *.sh
4
4
  recursive-include scripts *.sh *.py
5
5
  recursive-include frozen *.md *.sha256
6
6
  prune .github
7
7
  prune tests
8
+ prune examples/*/runtime
8
9
  global-exclude __pycache__
9
10
  global-exclude *.py[cod]
10
11
  global-exclude *.index *.npy *.npz *.sqlite *.sqlite3 *.db
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: embedflow
3
- Version: 0.1.1
3
+ Version: 0.2.0
4
4
  Summary: Progressive embedding-model migration over existing vector indexes.
5
5
  Author: Arnav Srivastav
6
6
  License-Expression: AGPL-3.0-only
@@ -8,7 +8,7 @@ Project-URL: Homepage, https://embedflow.org
8
8
  Project-URL: Repository, https://github.com/arnsri33/embedflow
9
9
  Project-URL: Documentation, https://github.com/arnsri33/embedflow#readme
10
10
  Project-URL: Issues, https://github.com/arnsri33/embedflow/issues
11
- Keywords: embeddings,vector-search,rag,information-retrieval,faiss,qdrant
11
+ Keywords: embeddings,vector-search,rag,information-retrieval,faiss,qdrant,pgvector
12
12
  Classifier: Development Status :: 3 - Alpha
13
13
  Classifier: Intended Audience :: Developers
14
14
  Classifier: Intended Audience :: Science/Research
@@ -27,9 +27,11 @@ Provides-Extra: faiss
27
27
  Requires-Dist: faiss-cpu>=1.8.0; extra == "faiss"
28
28
  Provides-Extra: qdrant
29
29
  Requires-Dist: qdrant-client>=1.9; extra == "qdrant"
30
+ Provides-Extra: pgvector
31
+ Requires-Dist: psycopg[binary]>=3.2; extra == "pgvector"
30
32
  Provides-Extra: models
31
- Requires-Dist: huggingface-hub>=0.20; extra == "models"
32
- Requires-Dist: transformers>=4.45; extra == "models"
33
+ Requires-Dist: huggingface-hub<1.0,>=0.34; extra == "models"
34
+ Requires-Dist: transformers<5.0,>=4.45; extra == "models"
33
35
  Requires-Dist: sentence-transformers>=3.0; extra == "models"
34
36
  Requires-Dist: torch>=2.2; extra == "models"
35
37
  Provides-Extra: dashboard
@@ -39,8 +41,9 @@ Requires-Dist: pydantic>=2.0; extra == "dashboard"
39
41
  Provides-Extra: all
40
42
  Requires-Dist: faiss-cpu>=1.8.0; extra == "all"
41
43
  Requires-Dist: qdrant-client>=1.9; extra == "all"
42
- Requires-Dist: huggingface-hub>=0.20; extra == "all"
43
- Requires-Dist: transformers>=4.45; extra == "all"
44
+ Requires-Dist: psycopg[binary]>=3.2; extra == "all"
45
+ Requires-Dist: huggingface-hub<1.0,>=0.34; extra == "all"
46
+ Requires-Dist: transformers<5.0,>=4.45; extra == "all"
44
47
  Requires-Dist: sentence-transformers>=3.0; extra == "all"
45
48
  Requires-Dist: torch>=2.2; extra == "all"
46
49
  Requires-Dist: fastapi>=0.100; extra == "all"
@@ -59,7 +62,7 @@ Dynamic: license-file
59
62
  EmbedFlow lets a new embedding model serve over candidates from an existing
60
63
  vector index while target document vectors are materialized progressively. It
61
64
  supports migration analysis, persistent caching, background work, FAISS,
62
- Qdrant, a CLI, and FastAPI.
65
+ Qdrant, pgvector, a CLI, and FastAPI.
63
66
 
64
67
  The full project README and architecture diagram are on
65
68
  <https://github.com/arnsri33/embedflow>.
@@ -76,6 +79,12 @@ For FAISS and the dashboard:
76
79
  python -m pip install "embedflow[faiss,dashboard]"
77
80
  ```
78
81
 
82
+ For an existing PostgreSQL/pgvector table:
83
+
84
+ ```bash
85
+ python -m pip install "embedflow[pgvector]"
86
+ ```
87
+
79
88
  Qdrant and model-runtime extras are documented in the
80
89
  [installation guide](https://github.com/arnsri33/embedflow/blob/main/docs/installation.md).
81
90
  For model-backed analysis, install `embedflow[faiss,models,dashboard]`.
@@ -176,9 +185,11 @@ for definitions and reproduction details.
176
185
  | --- | --- |
177
186
  | FAISS | Supported |
178
187
  | Qdrant | Supported |
188
+ | pgvector | Supported |
179
189
 
180
- See the [FAISS guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md)
181
- and [Qdrant guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/qdrant.md).
190
+ See the [FAISS guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md),
191
+ [Qdrant guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/qdrant.md),
192
+ and [pgvector guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/pgvector.md).
182
193
 
183
194
  ## CLI
184
195
 
@@ -188,7 +199,7 @@ embedflow analyze --help
188
199
  embedflow serve --config ./embedflow.yaml
189
200
  embedflow status --config ./embedflow.yaml
190
201
  embedflow registry list
191
- embedflow economics --corpus-size 1000000000 --docs-per-second 106.98 --gpu-price 3.29
202
+ embedflow economics --corpus-size 1000000000 --docs-per-second 100 --gpu-price 3.29
192
203
  embedflow doctor --config ./embedflow.yaml
193
204
  ```
194
205
 
@@ -198,7 +209,7 @@ cover the remaining commands and endpoints.
198
209
 
199
210
  ## Status
200
211
 
201
- EmbedFlow v0.1.0 is an alpha release for research and early real-world
212
+ EmbedFlow v0.2.0 is an alpha release for research and early real-world
202
213
  testing. T2-v1 is an empirical finite-tail diagnostic, partial rankings can
203
214
  differ from fully warm target reranking, and ANN fidelity needs a reference
204
215
  comparison to audit.
@@ -7,7 +7,7 @@
7
7
  EmbedFlow lets a new embedding model serve over candidates from an existing
8
8
  vector index while target document vectors are materialized progressively. It
9
9
  supports migration analysis, persistent caching, background work, and serving
10
- through FAISS, Qdrant, a CLI, and FastAPI.
10
+ through FAISS, Qdrant, pgvector, a CLI, and FastAPI.
11
11
 
12
12
  [Quickstart](#try-it) · [Documentation](#documentation) · [Research](#research)
13
13
 
@@ -189,11 +189,13 @@ is measured separately and is `UNKNOWN` until an exact reference is supplied.
189
189
  | --- | --- |
190
190
  | FAISS | Supported |
191
191
  | Qdrant | Supported |
192
+ | pgvector | Supported |
192
193
 
193
194
  Backend-specific setup and examples:
194
195
 
195
196
  - [FAISS](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md)
196
197
  - [Qdrant](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/qdrant.md)
198
+ - [pgvector](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/pgvector.md)
197
199
  - [Adding a backend](https://github.com/arnsri33/embedflow/blob/main/CONTRIBUTING.md)
198
200
 
199
201
  ## CLI
@@ -204,7 +206,7 @@ embedflow analyze --help
204
206
  embedflow serve --config ./embedflow.yaml
205
207
  embedflow status --config ./embedflow.yaml
206
208
  embedflow registry list
207
- embedflow economics --corpus-size 1000000000 --docs-per-second 106.98 --gpu-price 3.29
209
+ embedflow economics --corpus-size 1000000000 --docs-per-second 100 --gpu-price 3.29
208
210
  embedflow doctor --config ./embedflow.yaml
209
211
  ```
210
212
 
@@ -228,7 +230,7 @@ OpenAPI documentation; see
228
230
 
229
231
  ## Status
230
232
 
231
- EmbedFlow v0.1.0 is an alpha release for research and early real-world
233
+ EmbedFlow v0.2.0 is an alpha release for research and early real-world
232
234
  testing.
233
235
 
234
236
  - T2-v1 reports an empirical finite-tail diagnostic.
@@ -5,7 +5,7 @@
5
5
  EmbedFlow lets a new embedding model serve over candidates from an existing
6
6
  vector index while target document vectors are materialized progressively. It
7
7
  supports migration analysis, persistent caching, background work, FAISS,
8
- Qdrant, a CLI, and FastAPI.
8
+ Qdrant, pgvector, a CLI, and FastAPI.
9
9
 
10
10
  The full project README and architecture diagram are on
11
11
  <https://github.com/arnsri33/embedflow>.
@@ -22,6 +22,12 @@ For FAISS and the dashboard:
22
22
  python -m pip install "embedflow[faiss,dashboard]"
23
23
  ```
24
24
 
25
+ For an existing PostgreSQL/pgvector table:
26
+
27
+ ```bash
28
+ python -m pip install "embedflow[pgvector]"
29
+ ```
30
+
25
31
  Qdrant and model-runtime extras are documented in the
26
32
  [installation guide](https://github.com/arnsri33/embedflow/blob/main/docs/installation.md).
27
33
  For model-backed analysis, install `embedflow[faiss,models,dashboard]`.
@@ -122,9 +128,11 @@ for definitions and reproduction details.
122
128
  | --- | --- |
123
129
  | FAISS | Supported |
124
130
  | Qdrant | Supported |
131
+ | pgvector | Supported |
125
132
 
126
- See the [FAISS guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md)
127
- and [Qdrant guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/qdrant.md).
133
+ See the [FAISS guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md),
134
+ [Qdrant guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/qdrant.md),
135
+ and [pgvector guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/pgvector.md).
128
136
 
129
137
  ## CLI
130
138
 
@@ -134,7 +142,7 @@ embedflow analyze --help
134
142
  embedflow serve --config ./embedflow.yaml
135
143
  embedflow status --config ./embedflow.yaml
136
144
  embedflow registry list
137
- embedflow economics --corpus-size 1000000000 --docs-per-second 106.98 --gpu-price 3.29
145
+ embedflow economics --corpus-size 1000000000 --docs-per-second 100 --gpu-price 3.29
138
146
  embedflow doctor --config ./embedflow.yaml
139
147
  ```
140
148
 
@@ -144,7 +152,7 @@ cover the remaining commands and endpoints.
144
152
 
145
153
  ## Status
146
154
 
147
- EmbedFlow v0.1.0 is an alpha release for research and early real-world
155
+ EmbedFlow v0.2.0 is an alpha release for research and early real-world
148
156
  testing. T2-v1 is an empirical finite-tail diagnostic, partial rankings can
149
157
  differ from fully warm target reranking, and ANN fidelity needs a reference
150
158
  comparison to audit.
@@ -13,8 +13,8 @@ Interactive OpenAPI documentation is available at
13
13
 
14
14
  | Method | Path | Purpose |
15
15
  | --- | --- | --- |
16
- | GET | `/health` | Process and backend health |
17
- | GET | `/status` | Cache, queue, model, and migration state |
16
+ | GET | `/health` | Lightweight process liveness |
17
+ | GET | `/status` | Cache, queue, model, migration, and backend metadata |
18
18
  | POST | `/search` | Source retrieval and target reranking |
19
19
  | POST | `/analyze` | Run or retrieve migration analysis |
20
20
  | POST | `/prewarm` | Queue document materialization |
@@ -48,6 +48,10 @@ The response includes the result list and migration fields such as:
48
48
  request. A partial response scores the available target vectors; it can differ
49
49
  from the fully warm ranking.
50
50
 
51
+ `/status` includes safe backend metadata. For pgvector this names the
52
+ schema/table and vector contract without returning the DSN or credentials;
53
+ `/health` remains a compact liveness response for probes and load balancers.
54
+
51
55
  ## Errors
52
56
 
53
57
  Validation errors use FastAPI/Pydantic's normal JSON response. Backend,
@@ -52,7 +52,7 @@ Registry matching uses model contracts and corpus identity. See
52
52
  ```bash
53
53
  embedflow economics \
54
54
  --corpus-size 1000000000 \
55
- --docs-per-second 106.98 \
55
+ --docs-per-second 100 \
56
56
  --gpu-price 3.29
57
57
  embedflow doctor --config ./embedflow.yaml
58
58
  embedflow demo
@@ -14,12 +14,14 @@ target:
14
14
  device: cuda
15
15
 
16
16
  index:
17
- backend: faiss # faiss or qdrant
17
+ backend: faiss # faiss, qdrant, or pgvector
18
18
  path: ./legacy.index
19
19
  ids: ./legacy.index.ids.json # FAISS sidecar
20
20
  metric: cosine
21
21
  nprobe: 64
22
22
  # Qdrant fields: url, collection, vector_name, api_key_env
23
+ # pgvector fields: dsn_env, schema, table, id_column, vector_column, text_column
24
+ # hnsw_ef_search, ivfflat_probes
23
25
 
24
26
  documents:
25
27
  path: ./documents.jsonl
@@ -78,6 +80,14 @@ EMBEDFLOW_INDEX_URL
78
80
  EMBEDFLOW_INDEX_COLLECTION
79
81
  EMBEDFLOW_INDEX_VECTOR_NAME
80
82
  EMBEDFLOW_QDRANT_API_KEY_ENV
83
+ EMBEDFLOW_PGVECTOR_DSN_ENV
84
+ EMBEDFLOW_PGVECTOR_SCHEMA
85
+ EMBEDFLOW_PGVECTOR_TABLE
86
+ EMBEDFLOW_PGVECTOR_ID_COLUMN
87
+ EMBEDFLOW_PGVECTOR_VECTOR_COLUMN
88
+ EMBEDFLOW_PGVECTOR_TEXT_COLUMN
89
+ EMBEDFLOW_PGVECTOR_HNSW_EF_SEARCH
90
+ EMBEDFLOW_PGVECTOR_IVFFLAT_PROBES
81
91
  EMBEDFLOW_INDEX_NPROBE
82
92
  EMBEDFLOW_DOCUMENTS_PATH
83
93
  EMBEDFLOW_CACHE_PATH
@@ -6,7 +6,7 @@ throughput and GPU price supplied by the user:
6
6
  ```bash
7
7
  embedflow economics \
8
8
  --corpus-size 1000000000 \
9
- --docs-per-second 106.98 \
9
+ --docs-per-second 100 \
10
10
  --gpu-price 3.29
11
11
  ```
12
12
 
@@ -30,6 +30,7 @@ python -m pip install -e .
30
30
  | --- | --- |
31
31
  | `faiss` | FAISS source-index adapter |
32
32
  | `qdrant` | Qdrant client and adapter |
33
+ | `pgvector` | Psycopg 3 binary driver and pgvector adapter |
33
34
  | `models` | PyTorch, Transformers, Sentence Transformers, and Hub client |
34
35
  | `dashboard` | FastAPI, Uvicorn, and Pydantic |
35
36
  | `dev` | Pytest, Ruff, and build tooling |
@@ -52,6 +53,12 @@ with `index.url`; a user-owned cloud or remote server can use an API key named
52
53
  by `index.api_key_env`. Keep the key in the environment. See
53
54
  [`integrations/qdrant.md`](integrations/qdrant.md).
54
55
 
56
+ ## PostgreSQL / pgvector
57
+
58
+ Install the optional adapter with `python -m pip install "embedflow[pgvector]"`.
59
+ The adapter connects to an existing table and reads the DSN from the
60
+ environment; see [`integrations/pgvector.md`](integrations/pgvector.md).
61
+
55
62
  ## CPU and GPU
56
63
 
57
64
  The deterministic demo runs on CPU. Real model serving accepts `--device cpu`
@@ -0,0 +1,140 @@
1
+ # PostgreSQL / pgvector
2
+
3
+ EmbedFlow can read candidates from an existing PostgreSQL table that uses the
4
+ [pgvector](https://github.com/pgvector/pgvector) extension. The adapter uses
5
+ the shared `VectorIndex` interface, so analysis, serving, cache warming, and
6
+ the API follow the same path as FAISS and Qdrant.
7
+
8
+ ## Install
9
+
10
+ ```bash
11
+ python -m pip install "embedflow[pgvector]"
12
+ ```
13
+
14
+ The extra installs Psycopg 3 with its binary distribution. The base
15
+ `embedflow` install does not import or require PostgreSQL dependencies.
16
+
17
+ ## Connection and table layout
18
+
19
+ Keep the DSN in an environment variable:
20
+
21
+ ```bash
22
+ export EMBEDFLOW_PGVECTOR_DSN='postgresql://user:password@localhost:5432/app'
23
+ ```
24
+
25
+ Example configuration:
26
+
27
+ ```yaml
28
+ source:
29
+ model: sentence-transformers/all-MiniLM-L6-v2
30
+ device: cpu
31
+
32
+ target:
33
+ model: Qwen/Qwen3-Embedding-0.6B
34
+ device: cpu
35
+
36
+ index:
37
+ backend: pgvector
38
+ dsn_env: EMBEDFLOW_PGVECTOR_DSN
39
+ schema: public
40
+ table: documents
41
+ id_column: id
42
+ vector_column: embedding
43
+ text_column: content
44
+ metric: cosine
45
+ # hnsw_ef_search: 100
46
+ # ivfflat_probes: 20
47
+
48
+ documents:
49
+ # Omit this file when text_column is present in the table. A separate
50
+ # JSONL document store can be supplied when the table stores IDs only.
51
+ path: ./documents.jsonl
52
+ id_field: id
53
+ text_field: text
54
+ ```
55
+
56
+ When `documents.path` does not exist, EmbedFlow resolves candidate text from
57
+ `text_column` in the configured table. If a JSONL file exists, it remains the
58
+ document store and the pgvector table is used for vectors and IDs. The adapter
59
+ only reads the table during ordinary `analyze`, `serve`, `search`, `status`,
60
+ and `audit-index` operations; it does not create extensions, indexes, tables,
61
+ or modify rows.
62
+
63
+ The ID column may be an integer, bigint, UUID, or text type. IDs are normalized
64
+ to EmbedFlow's canonical string representation without numeric coercion.
65
+ The configured vector dimension is checked against `vector(n)` where available
66
+ and against a bounded non-null row for unbounded `vector` columns.
67
+
68
+ ## Metrics and score direction
69
+
70
+ Supported metrics are `cosine`, `l2`/`euclidean`, and
71
+ `inner_product`/`dot`. pgvector's distance operators are converted to the
72
+ EmbedFlow convention where larger scores rank first:
73
+
74
+ | EmbedFlow metric | pgvector operator | Internal score |
75
+ | --- | --- | --- |
76
+ | cosine | `<=>` | `1 - distance` |
77
+ | l2 / euclidean | `<->` | `-distance` |
78
+ | inner_product / dot | `<#>` | `-distance` |
79
+
80
+ The last conversion accounts for pgvector's negative inner-product operator.
81
+
82
+ ## ANN settings
83
+
84
+ EmbedFlow never creates or changes an ANN index. If configured, `hnsw_ef_search`
85
+ and `ivfflat_probes` are applied with transaction-local `set_config` calls for
86
+ the retrieval query. Omit them to use the database/index defaults. The
87
+ `metadata()` and `audit-index` output reports configured settings and detected
88
+ valid HNSW/IVFFlat indexes when PostgreSQL exposes them.
89
+
90
+ ## Commands
91
+
92
+ ```bash
93
+ embedflow analyze --config embedflow.yaml
94
+ embedflow serve --config embedflow.yaml
95
+ embedflow search --config embedflow.yaml "what causes auroras?"
96
+ embedflow status --config embedflow.yaml
97
+ embedflow audit-index --config embedflow.yaml
98
+ ```
99
+
100
+ `audit-index` checks table and column presence, pgvector type and dimension,
101
+ NULL vectors, duplicate canonical IDs, a retrieval probe, and available ANN
102
+ metadata. A successful connection is not an ANN recall audit; use an exact
103
+ reference comparison when fidelity matters.
104
+
105
+ ## Local example
106
+
107
+ The repository includes a small Docker example that creates a disposable
108
+ pgvector table and index:
109
+
110
+ ```bash
111
+ cd examples/pgvector
112
+ docker compose up -d
113
+ export EMBEDFLOW_PGVECTOR_DSN='postgresql://embedflow:embedflow@localhost:5432/embedflow'
114
+ ./run_demo.sh
115
+ ```
116
+
117
+ The example is for local testing only. It contains no credentials for a real
118
+ service and does not represent a production database configuration.
119
+
120
+ ## Troubleshooting
121
+
122
+ - `pgvector support requires psycopg`: install `embedflow[pgvector]` in the
123
+ active environment.
124
+ - `DSN not found`: export the variable named by `index.dsn_env`.
125
+ - `table ... was not found`: check the schema/table spelling and database.
126
+ - `is not a pgvector column`: install/enable the extension in the database and
127
+ point `vector_column` at a `vector` column.
128
+ - Dimension mismatch: set `source.dimension` to the encoder's dimension and
129
+ verify the table's vector type.
130
+ - Authentication errors are redacted in EmbedFlow output; inspect PostgreSQL
131
+ server logs without pasting passwords into issue reports.
132
+
133
+ ## Limitations
134
+
135
+ The adapter currently exposes the common table layout and no arbitrary SQL
136
+ filters. Metadata filtering will follow the shared backend interface when that
137
+ interface gains a portable filter contract. Connection operations on one
138
+ adapter are serialized; applications needing a larger pool can create several
139
+ sessions behind their own pool. ANN health is reported as `UNKNOWN` unless a
140
+ reference comparison is run.
@@ -1,6 +1,6 @@
1
1
  # Limitations and release scope
2
2
 
3
- EmbedFlow v0.1.0 is an alpha release for research and early real-world
3
+ EmbedFlow v0.2.0 is an alpha release for research and early real-world
4
4
  testing. The serving path is designed to make migration experiments concrete;
5
5
  production rollout still requires application-specific validation.
6
6
 
@@ -18,8 +18,8 @@ python -m twine check dist/*
18
18
  Inspect both archives before uploading:
19
19
 
20
20
  ```bash
21
- unzip -l dist/embedflow-0.1.0-py3-none-any.whl
22
- tar -tzf dist/embedflow-0.1.0.tar.gz
21
+ unzip -l dist/embedflow-0.2.0-py3-none-any.whl
22
+ tar -tzf dist/embedflow-0.2.0.tar.gz
23
23
  sha256sum dist/*
24
24
  ```
25
25
 
@@ -32,7 +32,7 @@ Test the wheel outside the source tree:
32
32
  ```bash
33
33
  python -m venv /tmp/embedflow-wheel-test
34
34
  /tmp/embedflow-wheel-test/bin/python -m pip install --upgrade pip
35
- /tmp/embedflow-wheel-test/bin/python -m pip install dist/embedflow-0.1.0-py3-none-any.whl
35
+ /tmp/embedflow-wheel-test/bin/python -m pip install dist/embedflow-0.2.0-py3-none-any.whl
36
36
  cd /tmp
37
37
  /tmp/embedflow-wheel-test/bin/python -c "import embedflow; print(embedflow.__version__)"
38
38
  /tmp/embedflow-wheel-test/bin/embedflow --help
@@ -65,7 +65,7 @@ python -m venv /tmp/embedflow-testpypi
65
65
  /tmp/embedflow-testpypi/bin/python -m pip install \
66
66
  --index-url https://test.pypi.org/simple/ \
67
67
  --extra-index-url https://pypi.org/simple/ \
68
- embedflow==0.1.0
68
+ embedflow==0.2.0
69
69
  cd /tmp
70
70
  /tmp/embedflow-testpypi/bin/python -c "import embedflow; print(embedflow.__version__)"
71
71
  /tmp/embedflow-testpypi/bin/embedflow --help
@@ -79,11 +79,11 @@ Test optional integrations in a second clean environment:
79
79
  /tmp/embedflow-testpypi/bin/python -m pip install \
80
80
  --index-url https://test.pypi.org/simple/ \
81
81
  --extra-index-url https://pypi.org/simple/ \
82
- "embedflow[faiss,dashboard]==0.1.0"
82
+ "embedflow[faiss,dashboard]==0.2.0"
83
83
  ```
84
84
 
85
85
  If the same filename already exists on TestPyPI, use a pre-release such as
86
- `0.1.0rc1` for the TestPyPI-only trial. Keep production `0.1.0` unchanged.
86
+ `0.2.0rc1` for the TestPyPI-only trial. Keep production `0.2.0` unchanged.
87
87
 
88
88
  ## Trusted Publishing configuration
89
89
 
@@ -115,7 +115,7 @@ above keeps the test step explicit.
115
115
  3. Run the final release gate and review the generated report.
116
116
  4. Configure the PyPI pending publisher and protected `pypi` environment.
117
117
  5. Create a Git tag and GitHub Release for the exact package version, for
118
- example `v0.1.0`.
118
+ example `v0.2.0`.
119
119
  6. Approve the `pypi` environment when the release workflow is ready.
120
120
  7. Verify the files and metadata on PyPI.
121
121
  8. Install from production PyPI in a directory outside this checkout.
@@ -123,7 +123,7 @@ above keeps the test step explicit.
123
123
  The GitHub workflow builds once, validates the metadata and wheel, transfers
124
124
  those exact files as an artifact, and then publishes them. PyPI filenames are
125
125
  immutable, so a correction after publishing requires a new version such as
126
- `0.1.1`.
126
+ `0.2.1`.
127
127
 
128
128
  ## After production publication
129
129
 
@@ -1,12 +1,12 @@
1
1
  """EmbedFlow: progressive embedding-model migration for existing indexes."""
2
2
 
3
- __version__ = "0.1.1"
3
+ __version__ = "0.2.0"
4
4
 
5
5
  from .config import EmbedFlowConfig, load_config
6
6
 
7
7
 
8
8
  def migrate(*args, **kwargs):
9
- """Start progressive migration over an existing FAISS/Qdrant index.
9
+ """Start progressive migration over an existing FAISS, Qdrant, or pgvector index.
10
10
 
11
11
  Imported lazily to keep the lightweight configuration package free of
12
12
  model-serving dependencies at import time. See ``embedflow.migration``
@@ -129,7 +129,7 @@ def analyze_migration(
129
129
  )
130
130
  if use_registry and registry.level != "EXACT REGISTRY MATCH":
131
131
  raise ValueError("--use-registry requires an EXACT REGISTRY MATCH")
132
- engine = open_engine(config_path, device=device, demo=demo, start_worker=False)
132
+ engine = open_engine(config_path, device=device, demo=demo, start_worker=False, documents=docs)
133
133
  result = run_probe(
134
134
  engine.source_model,
135
135
  engine.target_model,
@@ -185,6 +185,10 @@ def analyze_migration(
185
185
  finally:
186
186
  if engine is not None:
187
187
  engine.close()
188
+ elif 'docs' in locals():
189
+ close_documents = getattr(docs, "close", None)
190
+ if callable(close_documents):
191
+ close_documents()
188
192
  if temporary_config is not None:
189
193
  temporary_config.unlink(missing_ok=True)
190
194