embedflow 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {embedflow-0.1.1 → embedflow-0.2.0}/CHANGELOG.md +8 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/CITATION.cff +1 -1
- {embedflow-0.1.1 → embedflow-0.2.0}/MANIFEST.in +2 -1
- {embedflow-0.1.1 → embedflow-0.2.0}/PKG-INFO +22 -11
- {embedflow-0.1.1 → embedflow-0.2.0}/README.md +5 -3
- {embedflow-0.1.1 → embedflow-0.2.0}/README_PYPI.md +13 -5
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/api.md +6 -2
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/cli.md +1 -1
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/configuration.md +11 -1
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/economics.md +1 -1
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/installation.md +7 -0
- embedflow-0.2.0/docs/integrations/pgvector.md +140 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/limitations.md +1 -1
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/releasing.md +8 -8
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/__init__.py +2 -2
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/analysis.py +5 -1
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/cli.py +117 -30
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/config.py +40 -5
- embedflow-0.2.0/embedflow/indexes/__init__.py +9 -0
- embedflow-0.2.0/embedflow/indexes/pgvector_backend.py +685 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/facade.py +61 -12
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/materializer.py +18 -6
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/runtime.py +41 -6
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/api.py +13 -4
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/engine.py +8 -2
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/PKG-INFO +22 -11
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/SOURCES.txt +10 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/requires.txt +8 -4
- embedflow-0.2.0/examples/pgvector/README.md +28 -0
- embedflow-0.2.0/examples/pgvector/build_index.py +57 -0
- embedflow-0.2.0/examples/pgvector/compose.yaml +20 -0
- embedflow-0.2.0/examples/pgvector/embedflow.yaml +48 -0
- embedflow-0.2.0/examples/pgvector/init.sql +9 -0
- embedflow-0.2.0/examples/pgvector/queries.jsonl +4 -0
- embedflow-0.2.0/examples/pgvector/run_demo.sh +11 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/pyproject.toml +7 -5
- {embedflow-0.1.1 → embedflow-0.2.0}/scripts/release_gate.py +19 -2
- embedflow-0.2.0/scripts/validate_pgvector_10k.py +812 -0
- embedflow-0.1.1/embedflow/indexes/__init__.py +0 -5
- {embedflow-0.1.1 → embedflow-0.2.0}/CONTRIBUTING.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/LICENSE +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/SECURITY.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/assets/README.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/assets/candidate-gap-example.svg +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/assets/dashboard-screenshot.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/assets/terminal-demo.txt +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/concepts.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/contributing-benchmarks.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/integrations/faiss.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/integrations/qdrant.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/methodology.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/quickstart.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/docs/registry.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/__main__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/cache/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/cache/base.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/cache/persistent_cache.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/candidate_gap.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/containment.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/evaluate.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/metrics.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/migration_depth.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/probe.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/report.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/compatibility/t2.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/benchmark_profiles.jsonl +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/checksums.sha256 +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/migrations.jsonl +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/registry_manifest.json +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/research_summaries.json +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/data/registry/schema_version.json +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/frozen/T2_V1_FROZEN_SPEC.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/frozen/T2_V1_FROZEN_SPEC.sha256 +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/indexes/base.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/indexes/faiss_backend.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/indexes/qdrant_backend.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/metrics/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/metrics/latency.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/compatibility.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/planner.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/migration/state.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/models/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/models/base.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/models/huggingface.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/registry/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/registry/loader.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/registry/matcher.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/registry/schema.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/factory.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow/serving/schemas.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/dependency_links.txt +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/entry_points.txt +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/embedflow.egg-info/top_level.txt +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/faiss/README.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/faiss/documents.jsonl +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/faiss/embedflow.yaml +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/faiss/queries.jsonl +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/README.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/build_index.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/documents.jsonl +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/embedflow.yaml +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/qdrant/queries.jsonl +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/README.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/documents.jsonl +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/embedflow.yaml +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/qrels.json +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/examples/research_analysis/queries.jsonl +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/frozen/T2_V1_FROZEN_SPEC.md +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/frozen/T2_V1_FROZEN_SPEC.sha256 +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/requirements-dev.txt +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/requirements.txt +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/scripts/real_qdrant_smoke.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/scripts/run_demo.sh +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/scripts/run_tests.sh +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/setup.cfg +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/src/__init__.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/src/embed.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/src/probe_features.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/src/storage.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/src/t2_v1.py +0 -0
- {embedflow-0.1.1 → embedflow-0.2.0}/src/utils.py +0 -0
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## v0.2.0 — pgvector backend
|
|
4
|
+
|
|
5
|
+
- Added a read-only pgvector backend for existing PostgreSQL vector tables.
|
|
6
|
+
- Added cosine, Euclidean/L2, and inner-product retrieval with safe SQL
|
|
7
|
+
identifier composition.
|
|
8
|
+
- Added pgvector table auditing, optional transaction-local HNSW/IVFFlat
|
|
9
|
+
settings, Docker example, and numerical/SQL-safety tests.
|
|
10
|
+
|
|
3
11
|
## v0.1.0 — initial public release
|
|
4
12
|
|
|
5
13
|
- Candidate-compatibility analysis and Mode A evaluation workflows.
|
|
@@ -2,7 +2,7 @@ cff-version: 1.2.0
|
|
|
2
2
|
title: "EmbedFlow: Upgrading Legacy Embeddings Without Full Upfront Re-Embedding"
|
|
3
3
|
message: "If EmbedFlow contributes to your work, please cite this software release."
|
|
4
4
|
type: software
|
|
5
|
-
version: 0.
|
|
5
|
+
version: 0.2.0
|
|
6
6
|
date-released: 2026-09-06
|
|
7
7
|
repository-code: "https://github.com/arnsri33/embedflow"
|
|
8
8
|
url: "https://github.com/arnsri33/embedflow"
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
include README.md README_PYPI.md LICENSE CHANGELOG.md CONTRIBUTING.md SECURITY.md CITATION.cff requirements.txt requirements-dev.txt
|
|
2
2
|
recursive-include docs *
|
|
3
|
-
recursive-include examples *.md *.yaml *.json *.jsonl *.py
|
|
3
|
+
recursive-include examples *.md *.yaml *.json *.jsonl *.py *.sql *.sh
|
|
4
4
|
recursive-include scripts *.sh *.py
|
|
5
5
|
recursive-include frozen *.md *.sha256
|
|
6
6
|
prune .github
|
|
7
7
|
prune tests
|
|
8
|
+
prune examples/*/runtime
|
|
8
9
|
global-exclude __pycache__
|
|
9
10
|
global-exclude *.py[cod]
|
|
10
11
|
global-exclude *.index *.npy *.npz *.sqlite *.sqlite3 *.db
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: embedflow
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Progressive embedding-model migration over existing vector indexes.
|
|
5
5
|
Author: Arnav Srivastav
|
|
6
6
|
License-Expression: AGPL-3.0-only
|
|
@@ -8,7 +8,7 @@ Project-URL: Homepage, https://embedflow.org
|
|
|
8
8
|
Project-URL: Repository, https://github.com/arnsri33/embedflow
|
|
9
9
|
Project-URL: Documentation, https://github.com/arnsri33/embedflow#readme
|
|
10
10
|
Project-URL: Issues, https://github.com/arnsri33/embedflow/issues
|
|
11
|
-
Keywords: embeddings,vector-search,rag,information-retrieval,faiss,qdrant
|
|
11
|
+
Keywords: embeddings,vector-search,rag,information-retrieval,faiss,qdrant,pgvector
|
|
12
12
|
Classifier: Development Status :: 3 - Alpha
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
14
14
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -27,9 +27,11 @@ Provides-Extra: faiss
|
|
|
27
27
|
Requires-Dist: faiss-cpu>=1.8.0; extra == "faiss"
|
|
28
28
|
Provides-Extra: qdrant
|
|
29
29
|
Requires-Dist: qdrant-client>=1.9; extra == "qdrant"
|
|
30
|
+
Provides-Extra: pgvector
|
|
31
|
+
Requires-Dist: psycopg[binary]>=3.2; extra == "pgvector"
|
|
30
32
|
Provides-Extra: models
|
|
31
|
-
Requires-Dist: huggingface-hub
|
|
32
|
-
Requires-Dist: transformers
|
|
33
|
+
Requires-Dist: huggingface-hub<1.0,>=0.34; extra == "models"
|
|
34
|
+
Requires-Dist: transformers<5.0,>=4.45; extra == "models"
|
|
33
35
|
Requires-Dist: sentence-transformers>=3.0; extra == "models"
|
|
34
36
|
Requires-Dist: torch>=2.2; extra == "models"
|
|
35
37
|
Provides-Extra: dashboard
|
|
@@ -39,8 +41,9 @@ Requires-Dist: pydantic>=2.0; extra == "dashboard"
|
|
|
39
41
|
Provides-Extra: all
|
|
40
42
|
Requires-Dist: faiss-cpu>=1.8.0; extra == "all"
|
|
41
43
|
Requires-Dist: qdrant-client>=1.9; extra == "all"
|
|
42
|
-
Requires-Dist:
|
|
43
|
-
Requires-Dist:
|
|
44
|
+
Requires-Dist: psycopg[binary]>=3.2; extra == "all"
|
|
45
|
+
Requires-Dist: huggingface-hub<1.0,>=0.34; extra == "all"
|
|
46
|
+
Requires-Dist: transformers<5.0,>=4.45; extra == "all"
|
|
44
47
|
Requires-Dist: sentence-transformers>=3.0; extra == "all"
|
|
45
48
|
Requires-Dist: torch>=2.2; extra == "all"
|
|
46
49
|
Requires-Dist: fastapi>=0.100; extra == "all"
|
|
@@ -59,7 +62,7 @@ Dynamic: license-file
|
|
|
59
62
|
EmbedFlow lets a new embedding model serve over candidates from an existing
|
|
60
63
|
vector index while target document vectors are materialized progressively. It
|
|
61
64
|
supports migration analysis, persistent caching, background work, FAISS,
|
|
62
|
-
Qdrant, a CLI, and FastAPI.
|
|
65
|
+
Qdrant, pgvector, a CLI, and FastAPI.
|
|
63
66
|
|
|
64
67
|
The full project README and architecture diagram are on
|
|
65
68
|
<https://github.com/arnsri33/embedflow>.
|
|
@@ -76,6 +79,12 @@ For FAISS and the dashboard:
|
|
|
76
79
|
python -m pip install "embedflow[faiss,dashboard]"
|
|
77
80
|
```
|
|
78
81
|
|
|
82
|
+
For an existing PostgreSQL/pgvector table:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
python -m pip install "embedflow[pgvector]"
|
|
86
|
+
```
|
|
87
|
+
|
|
79
88
|
Qdrant and model-runtime extras are documented in the
|
|
80
89
|
[installation guide](https://github.com/arnsri33/embedflow/blob/main/docs/installation.md).
|
|
81
90
|
For model-backed analysis, install `embedflow[faiss,models,dashboard]`.
|
|
@@ -176,9 +185,11 @@ for definitions and reproduction details.
|
|
|
176
185
|
| --- | --- |
|
|
177
186
|
| FAISS | Supported |
|
|
178
187
|
| Qdrant | Supported |
|
|
188
|
+
| pgvector | Supported |
|
|
179
189
|
|
|
180
|
-
See the [FAISS guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md)
|
|
181
|
-
|
|
190
|
+
See the [FAISS guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md),
|
|
191
|
+
[Qdrant guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/qdrant.md),
|
|
192
|
+
and [pgvector guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/pgvector.md).
|
|
182
193
|
|
|
183
194
|
## CLI
|
|
184
195
|
|
|
@@ -188,7 +199,7 @@ embedflow analyze --help
|
|
|
188
199
|
embedflow serve --config ./embedflow.yaml
|
|
189
200
|
embedflow status --config ./embedflow.yaml
|
|
190
201
|
embedflow registry list
|
|
191
|
-
embedflow economics --corpus-size 1000000000 --docs-per-second
|
|
202
|
+
embedflow economics --corpus-size 1000000000 --docs-per-second 100 --gpu-price 3.29
|
|
192
203
|
embedflow doctor --config ./embedflow.yaml
|
|
193
204
|
```
|
|
194
205
|
|
|
@@ -198,7 +209,7 @@ cover the remaining commands and endpoints.
|
|
|
198
209
|
|
|
199
210
|
## Status
|
|
200
211
|
|
|
201
|
-
EmbedFlow v0.
|
|
212
|
+
EmbedFlow v0.2.0 is an alpha release for research and early real-world
|
|
202
213
|
testing. T2-v1 is an empirical finite-tail diagnostic, partial rankings can
|
|
203
214
|
differ from fully warm target reranking, and ANN fidelity needs a reference
|
|
204
215
|
comparison to audit.
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
EmbedFlow lets a new embedding model serve over candidates from an existing
|
|
8
8
|
vector index while target document vectors are materialized progressively. It
|
|
9
9
|
supports migration analysis, persistent caching, background work, and serving
|
|
10
|
-
through FAISS, Qdrant, a CLI, and FastAPI.
|
|
10
|
+
through FAISS, Qdrant, pgvector, a CLI, and FastAPI.
|
|
11
11
|
|
|
12
12
|
[Quickstart](#try-it) · [Documentation](#documentation) · [Research](#research)
|
|
13
13
|
|
|
@@ -189,11 +189,13 @@ is measured separately and is `UNKNOWN` until an exact reference is supplied.
|
|
|
189
189
|
| --- | --- |
|
|
190
190
|
| FAISS | Supported |
|
|
191
191
|
| Qdrant | Supported |
|
|
192
|
+
| pgvector | Supported |
|
|
192
193
|
|
|
193
194
|
Backend-specific setup and examples:
|
|
194
195
|
|
|
195
196
|
- [FAISS](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md)
|
|
196
197
|
- [Qdrant](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/qdrant.md)
|
|
198
|
+
- [pgvector](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/pgvector.md)
|
|
197
199
|
- [Adding a backend](https://github.com/arnsri33/embedflow/blob/main/CONTRIBUTING.md)
|
|
198
200
|
|
|
199
201
|
## CLI
|
|
@@ -204,7 +206,7 @@ embedflow analyze --help
|
|
|
204
206
|
embedflow serve --config ./embedflow.yaml
|
|
205
207
|
embedflow status --config ./embedflow.yaml
|
|
206
208
|
embedflow registry list
|
|
207
|
-
embedflow economics --corpus-size 1000000000 --docs-per-second
|
|
209
|
+
embedflow economics --corpus-size 1000000000 --docs-per-second 100 --gpu-price 3.29
|
|
208
210
|
embedflow doctor --config ./embedflow.yaml
|
|
209
211
|
```
|
|
210
212
|
|
|
@@ -228,7 +230,7 @@ OpenAPI documentation; see
|
|
|
228
230
|
|
|
229
231
|
## Status
|
|
230
232
|
|
|
231
|
-
EmbedFlow v0.
|
|
233
|
+
EmbedFlow v0.2.0 is an alpha release for research and early real-world
|
|
232
234
|
testing.
|
|
233
235
|
|
|
234
236
|
- T2-v1 reports an empirical finite-tail diagnostic.
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
EmbedFlow lets a new embedding model serve over candidates from an existing
|
|
6
6
|
vector index while target document vectors are materialized progressively. It
|
|
7
7
|
supports migration analysis, persistent caching, background work, FAISS,
|
|
8
|
-
Qdrant, a CLI, and FastAPI.
|
|
8
|
+
Qdrant, pgvector, a CLI, and FastAPI.
|
|
9
9
|
|
|
10
10
|
The full project README and architecture diagram are on
|
|
11
11
|
<https://github.com/arnsri33/embedflow>.
|
|
@@ -22,6 +22,12 @@ For FAISS and the dashboard:
|
|
|
22
22
|
python -m pip install "embedflow[faiss,dashboard]"
|
|
23
23
|
```
|
|
24
24
|
|
|
25
|
+
For an existing PostgreSQL/pgvector table:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
python -m pip install "embedflow[pgvector]"
|
|
29
|
+
```
|
|
30
|
+
|
|
25
31
|
Qdrant and model-runtime extras are documented in the
|
|
26
32
|
[installation guide](https://github.com/arnsri33/embedflow/blob/main/docs/installation.md).
|
|
27
33
|
For model-backed analysis, install `embedflow[faiss,models,dashboard]`.
|
|
@@ -122,9 +128,11 @@ for definitions and reproduction details.
|
|
|
122
128
|
| --- | --- |
|
|
123
129
|
| FAISS | Supported |
|
|
124
130
|
| Qdrant | Supported |
|
|
131
|
+
| pgvector | Supported |
|
|
125
132
|
|
|
126
|
-
See the [FAISS guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md)
|
|
127
|
-
|
|
133
|
+
See the [FAISS guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/faiss.md),
|
|
134
|
+
[Qdrant guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/qdrant.md),
|
|
135
|
+
and [pgvector guide](https://github.com/arnsri33/embedflow/blob/main/docs/integrations/pgvector.md).
|
|
128
136
|
|
|
129
137
|
## CLI
|
|
130
138
|
|
|
@@ -134,7 +142,7 @@ embedflow analyze --help
|
|
|
134
142
|
embedflow serve --config ./embedflow.yaml
|
|
135
143
|
embedflow status --config ./embedflow.yaml
|
|
136
144
|
embedflow registry list
|
|
137
|
-
embedflow economics --corpus-size 1000000000 --docs-per-second
|
|
145
|
+
embedflow economics --corpus-size 1000000000 --docs-per-second 100 --gpu-price 3.29
|
|
138
146
|
embedflow doctor --config ./embedflow.yaml
|
|
139
147
|
```
|
|
140
148
|
|
|
@@ -144,7 +152,7 @@ cover the remaining commands and endpoints.
|
|
|
144
152
|
|
|
145
153
|
## Status
|
|
146
154
|
|
|
147
|
-
EmbedFlow v0.
|
|
155
|
+
EmbedFlow v0.2.0 is an alpha release for research and early real-world
|
|
148
156
|
testing. T2-v1 is an empirical finite-tail diagnostic, partial rankings can
|
|
149
157
|
differ from fully warm target reranking, and ANN fidelity needs a reference
|
|
150
158
|
comparison to audit.
|
|
@@ -13,8 +13,8 @@ Interactive OpenAPI documentation is available at
|
|
|
13
13
|
|
|
14
14
|
| Method | Path | Purpose |
|
|
15
15
|
| --- | --- | --- |
|
|
16
|
-
| GET | `/health` |
|
|
17
|
-
| GET | `/status` | Cache, queue, model, and
|
|
16
|
+
| GET | `/health` | Lightweight process liveness |
|
|
17
|
+
| GET | `/status` | Cache, queue, model, migration, and backend metadata |
|
|
18
18
|
| POST | `/search` | Source retrieval and target reranking |
|
|
19
19
|
| POST | `/analyze` | Run or retrieve migration analysis |
|
|
20
20
|
| POST | `/prewarm` | Queue document materialization |
|
|
@@ -48,6 +48,10 @@ The response includes the result list and migration fields such as:
|
|
|
48
48
|
request. A partial response scores the available target vectors; it can differ
|
|
49
49
|
from the fully warm ranking.
|
|
50
50
|
|
|
51
|
+
`/status` includes safe backend metadata. For pgvector this names the
|
|
52
|
+
schema/table and vector contract without returning the DSN or credentials;
|
|
53
|
+
`/health` remains a compact liveness response for probes and load balancers.
|
|
54
|
+
|
|
51
55
|
## Errors
|
|
52
56
|
|
|
53
57
|
Validation errors use FastAPI/Pydantic's normal JSON response. Backend,
|
|
@@ -52,7 +52,7 @@ Registry matching uses model contracts and corpus identity. See
|
|
|
52
52
|
```bash
|
|
53
53
|
embedflow economics \
|
|
54
54
|
--corpus-size 1000000000 \
|
|
55
|
-
--docs-per-second
|
|
55
|
+
--docs-per-second 100 \
|
|
56
56
|
--gpu-price 3.29
|
|
57
57
|
embedflow doctor --config ./embedflow.yaml
|
|
58
58
|
embedflow demo
|
|
@@ -14,12 +14,14 @@ target:
|
|
|
14
14
|
device: cuda
|
|
15
15
|
|
|
16
16
|
index:
|
|
17
|
-
backend: faiss # faiss or
|
|
17
|
+
backend: faiss # faiss, qdrant, or pgvector
|
|
18
18
|
path: ./legacy.index
|
|
19
19
|
ids: ./legacy.index.ids.json # FAISS sidecar
|
|
20
20
|
metric: cosine
|
|
21
21
|
nprobe: 64
|
|
22
22
|
# Qdrant fields: url, collection, vector_name, api_key_env
|
|
23
|
+
# pgvector fields: dsn_env, schema, table, id_column, vector_column, text_column
|
|
24
|
+
# hnsw_ef_search, ivfflat_probes
|
|
23
25
|
|
|
24
26
|
documents:
|
|
25
27
|
path: ./documents.jsonl
|
|
@@ -78,6 +80,14 @@ EMBEDFLOW_INDEX_URL
|
|
|
78
80
|
EMBEDFLOW_INDEX_COLLECTION
|
|
79
81
|
EMBEDFLOW_INDEX_VECTOR_NAME
|
|
80
82
|
EMBEDFLOW_QDRANT_API_KEY_ENV
|
|
83
|
+
EMBEDFLOW_PGVECTOR_DSN_ENV
|
|
84
|
+
EMBEDFLOW_PGVECTOR_SCHEMA
|
|
85
|
+
EMBEDFLOW_PGVECTOR_TABLE
|
|
86
|
+
EMBEDFLOW_PGVECTOR_ID_COLUMN
|
|
87
|
+
EMBEDFLOW_PGVECTOR_VECTOR_COLUMN
|
|
88
|
+
EMBEDFLOW_PGVECTOR_TEXT_COLUMN
|
|
89
|
+
EMBEDFLOW_PGVECTOR_HNSW_EF_SEARCH
|
|
90
|
+
EMBEDFLOW_PGVECTOR_IVFFLAT_PROBES
|
|
81
91
|
EMBEDFLOW_INDEX_NPROBE
|
|
82
92
|
EMBEDFLOW_DOCUMENTS_PATH
|
|
83
93
|
EMBEDFLOW_CACHE_PATH
|
|
@@ -30,6 +30,7 @@ python -m pip install -e .
|
|
|
30
30
|
| --- | --- |
|
|
31
31
|
| `faiss` | FAISS source-index adapter |
|
|
32
32
|
| `qdrant` | Qdrant client and adapter |
|
|
33
|
+
| `pgvector` | Psycopg 3 binary driver and pgvector adapter |
|
|
33
34
|
| `models` | PyTorch, Transformers, Sentence Transformers, and Hub client |
|
|
34
35
|
| `dashboard` | FastAPI, Uvicorn, and Pydantic |
|
|
35
36
|
| `dev` | Pytest, Ruff, and build tooling |
|
|
@@ -52,6 +53,12 @@ with `index.url`; a user-owned cloud or remote server can use an API key named
|
|
|
52
53
|
by `index.api_key_env`. Keep the key in the environment. See
|
|
53
54
|
[`integrations/qdrant.md`](integrations/qdrant.md).
|
|
54
55
|
|
|
56
|
+
## PostgreSQL / pgvector
|
|
57
|
+
|
|
58
|
+
Install the optional adapter with `python -m pip install "embedflow[pgvector]"`.
|
|
59
|
+
The adapter connects to an existing table and reads the DSN from the
|
|
60
|
+
environment; see [`integrations/pgvector.md`](integrations/pgvector.md).
|
|
61
|
+
|
|
55
62
|
## CPU and GPU
|
|
56
63
|
|
|
57
64
|
The deterministic demo runs on CPU. Real model serving accepts `--device cpu`
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# PostgreSQL / pgvector
|
|
2
|
+
|
|
3
|
+
EmbedFlow can read candidates from an existing PostgreSQL table that uses the
|
|
4
|
+
[pgvector](https://github.com/pgvector/pgvector) extension. The adapter uses
|
|
5
|
+
the shared `VectorIndex` interface, so analysis, serving, cache warming, and
|
|
6
|
+
the API follow the same path as FAISS and Qdrant.
|
|
7
|
+
|
|
8
|
+
## Install
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
python -m pip install "embedflow[pgvector]"
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
The extra installs Psycopg 3 with its binary distribution. The base
|
|
15
|
+
`embedflow` install does not import or require PostgreSQL dependencies.
|
|
16
|
+
|
|
17
|
+
## Connection and table layout
|
|
18
|
+
|
|
19
|
+
Keep the DSN in an environment variable:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
export EMBEDFLOW_PGVECTOR_DSN='postgresql://user:password@localhost:5432/app'
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Example configuration:
|
|
26
|
+
|
|
27
|
+
```yaml
|
|
28
|
+
source:
|
|
29
|
+
model: sentence-transformers/all-MiniLM-L6-v2
|
|
30
|
+
device: cpu
|
|
31
|
+
|
|
32
|
+
target:
|
|
33
|
+
model: Qwen/Qwen3-Embedding-0.6B
|
|
34
|
+
device: cpu
|
|
35
|
+
|
|
36
|
+
index:
|
|
37
|
+
backend: pgvector
|
|
38
|
+
dsn_env: EMBEDFLOW_PGVECTOR_DSN
|
|
39
|
+
schema: public
|
|
40
|
+
table: documents
|
|
41
|
+
id_column: id
|
|
42
|
+
vector_column: embedding
|
|
43
|
+
text_column: content
|
|
44
|
+
metric: cosine
|
|
45
|
+
# hnsw_ef_search: 100
|
|
46
|
+
# ivfflat_probes: 20
|
|
47
|
+
|
|
48
|
+
documents:
|
|
49
|
+
# Omit this file when text_column is present in the table. A separate
|
|
50
|
+
# JSONL document store can be supplied when the table stores IDs only.
|
|
51
|
+
path: ./documents.jsonl
|
|
52
|
+
id_field: id
|
|
53
|
+
text_field: text
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
When `documents.path` does not exist, EmbedFlow resolves candidate text from
|
|
57
|
+
`text_column` in the configured table. If a JSONL file exists, it remains the
|
|
58
|
+
document store and the pgvector table is used for vectors and IDs. The adapter
|
|
59
|
+
only reads the table during ordinary `analyze`, `serve`, `search`, `status`,
|
|
60
|
+
and `audit-index` operations; it does not create extensions, indexes, tables,
|
|
61
|
+
or modify rows.
|
|
62
|
+
|
|
63
|
+
The ID column may be an integer, bigint, UUID, or text type. IDs are normalized
|
|
64
|
+
to EmbedFlow's canonical string representation without numeric coercion.
|
|
65
|
+
The configured vector dimension is checked against `vector(n)` where available
|
|
66
|
+
and against a bounded non-null row for unbounded `vector` columns.
|
|
67
|
+
|
|
68
|
+
## Metrics and score direction
|
|
69
|
+
|
|
70
|
+
Supported metrics are `cosine`, `l2`/`euclidean`, and
|
|
71
|
+
`inner_product`/`dot`. pgvector's distance operators are converted to the
|
|
72
|
+
EmbedFlow convention where larger scores rank first:
|
|
73
|
+
|
|
74
|
+
| EmbedFlow metric | pgvector operator | Internal score |
|
|
75
|
+
| --- | --- | --- |
|
|
76
|
+
| cosine | `<=>` | `1 - distance` |
|
|
77
|
+
| l2 / euclidean | `<->` | `-distance` |
|
|
78
|
+
| inner_product / dot | `<#>` | `-distance` |
|
|
79
|
+
|
|
80
|
+
The last conversion accounts for pgvector's negative inner-product operator.
|
|
81
|
+
|
|
82
|
+
## ANN settings
|
|
83
|
+
|
|
84
|
+
EmbedFlow never creates or changes an ANN index. If configured, `hnsw_ef_search`
|
|
85
|
+
and `ivfflat_probes` are applied with transaction-local `set_config` calls for
|
|
86
|
+
the retrieval query. Omit them to use the database/index defaults. The
|
|
87
|
+
`metadata()` and `audit-index` output reports configured settings and detected
|
|
88
|
+
valid HNSW/IVFFlat indexes when PostgreSQL exposes them.
|
|
89
|
+
|
|
90
|
+
## Commands
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
embedflow analyze --config embedflow.yaml
|
|
94
|
+
embedflow serve --config embedflow.yaml
|
|
95
|
+
embedflow search --config embedflow.yaml "what causes auroras?"
|
|
96
|
+
embedflow status --config embedflow.yaml
|
|
97
|
+
embedflow audit-index --config embedflow.yaml
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
`audit-index` checks table and column presence, pgvector type and dimension,
|
|
101
|
+
NULL vectors, duplicate canonical IDs, a retrieval probe, and available ANN
|
|
102
|
+
metadata. A successful connection is not an ANN recall audit; use an exact
|
|
103
|
+
reference comparison when fidelity matters.
|
|
104
|
+
|
|
105
|
+
## Local example
|
|
106
|
+
|
|
107
|
+
The repository includes a small Docker example that creates a disposable
|
|
108
|
+
pgvector table and index:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
cd examples/pgvector
|
|
112
|
+
docker compose up -d
|
|
113
|
+
export EMBEDFLOW_PGVECTOR_DSN='postgresql://embedflow:embedflow@localhost:5432/embedflow'
|
|
114
|
+
./run_demo.sh
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
The example is for local testing only. It contains no credentials for a real
|
|
118
|
+
service and does not represent a production database configuration.
|
|
119
|
+
|
|
120
|
+
## Troubleshooting
|
|
121
|
+
|
|
122
|
+
- `pgvector support requires psycopg`: install `embedflow[pgvector]` in the
|
|
123
|
+
active environment.
|
|
124
|
+
- `DSN not found`: export the variable named by `index.dsn_env`.
|
|
125
|
+
- `table ... was not found`: check the schema/table spelling and database.
|
|
126
|
+
- `is not a pgvector column`: install/enable the extension in the database and
|
|
127
|
+
point `vector_column` at a `vector` column.
|
|
128
|
+
- Dimension mismatch: set `source.dimension` to the encoder's dimension and
|
|
129
|
+
verify the table's vector type.
|
|
130
|
+
- Authentication errors are redacted in EmbedFlow output; inspect PostgreSQL
|
|
131
|
+
server logs without pasting passwords into issue reports.
|
|
132
|
+
|
|
133
|
+
## Limitations
|
|
134
|
+
|
|
135
|
+
The adapter currently exposes the common table layout and no arbitrary SQL
|
|
136
|
+
filters. Metadata filtering will follow the shared backend interface when that
|
|
137
|
+
interface gains a portable filter contract. Connection operations on one
|
|
138
|
+
adapter are serialized; applications needing a larger pool can create several
|
|
139
|
+
sessions behind their own pool. ANN health is reported as `UNKNOWN` unless a
|
|
140
|
+
reference comparison is run.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Limitations and release scope
|
|
2
2
|
|
|
3
|
-
EmbedFlow v0.
|
|
3
|
+
EmbedFlow v0.2.0 is an alpha release for research and early real-world
|
|
4
4
|
testing. The serving path is designed to make migration experiments concrete;
|
|
5
5
|
production rollout still requires application-specific validation.
|
|
6
6
|
|
|
@@ -18,8 +18,8 @@ python -m twine check dist/*
|
|
|
18
18
|
Inspect both archives before uploading:
|
|
19
19
|
|
|
20
20
|
```bash
|
|
21
|
-
unzip -l dist/embedflow-0.
|
|
22
|
-
tar -tzf dist/embedflow-0.
|
|
21
|
+
unzip -l dist/embedflow-0.2.0-py3-none-any.whl
|
|
22
|
+
tar -tzf dist/embedflow-0.2.0.tar.gz
|
|
23
23
|
sha256sum dist/*
|
|
24
24
|
```
|
|
25
25
|
|
|
@@ -32,7 +32,7 @@ Test the wheel outside the source tree:
|
|
|
32
32
|
```bash
|
|
33
33
|
python -m venv /tmp/embedflow-wheel-test
|
|
34
34
|
/tmp/embedflow-wheel-test/bin/python -m pip install --upgrade pip
|
|
35
|
-
/tmp/embedflow-wheel-test/bin/python -m pip install dist/embedflow-0.
|
|
35
|
+
/tmp/embedflow-wheel-test/bin/python -m pip install dist/embedflow-0.2.0-py3-none-any.whl
|
|
36
36
|
cd /tmp
|
|
37
37
|
/tmp/embedflow-wheel-test/bin/python -c "import embedflow; print(embedflow.__version__)"
|
|
38
38
|
/tmp/embedflow-wheel-test/bin/embedflow --help
|
|
@@ -65,7 +65,7 @@ python -m venv /tmp/embedflow-testpypi
|
|
|
65
65
|
/tmp/embedflow-testpypi/bin/python -m pip install \
|
|
66
66
|
--index-url https://test.pypi.org/simple/ \
|
|
67
67
|
--extra-index-url https://pypi.org/simple/ \
|
|
68
|
-
embedflow==0.
|
|
68
|
+
embedflow==0.2.0
|
|
69
69
|
cd /tmp
|
|
70
70
|
/tmp/embedflow-testpypi/bin/python -c "import embedflow; print(embedflow.__version__)"
|
|
71
71
|
/tmp/embedflow-testpypi/bin/embedflow --help
|
|
@@ -79,11 +79,11 @@ Test optional integrations in a second clean environment:
|
|
|
79
79
|
/tmp/embedflow-testpypi/bin/python -m pip install \
|
|
80
80
|
--index-url https://test.pypi.org/simple/ \
|
|
81
81
|
--extra-index-url https://pypi.org/simple/ \
|
|
82
|
-
"embedflow[faiss,dashboard]==0.
|
|
82
|
+
"embedflow[faiss,dashboard]==0.2.0"
|
|
83
83
|
```
|
|
84
84
|
|
|
85
85
|
If the same filename already exists on TestPyPI, use a pre-release such as
|
|
86
|
-
`0.
|
|
86
|
+
`0.2.0rc1` for the TestPyPI-only trial. Keep production `0.2.0` unchanged.
|
|
87
87
|
|
|
88
88
|
## Trusted Publishing configuration
|
|
89
89
|
|
|
@@ -115,7 +115,7 @@ above keeps the test step explicit.
|
|
|
115
115
|
3. Run the final release gate and review the generated report.
|
|
116
116
|
4. Configure the PyPI pending publisher and protected `pypi` environment.
|
|
117
117
|
5. Create a Git tag and GitHub Release for the exact package version, for
|
|
118
|
-
example `v0.
|
|
118
|
+
example `v0.2.0`.
|
|
119
119
|
6. Approve the `pypi` environment when the release workflow is ready.
|
|
120
120
|
7. Verify the files and metadata on PyPI.
|
|
121
121
|
8. Install from production PyPI in a directory outside this checkout.
|
|
@@ -123,7 +123,7 @@ above keeps the test step explicit.
|
|
|
123
123
|
The GitHub workflow builds once, validates the metadata and wheel, transfers
|
|
124
124
|
those exact files as an artifact, and then publishes them. PyPI filenames are
|
|
125
125
|
immutable, so a correction after publishing requires a new version such as
|
|
126
|
-
`0.
|
|
126
|
+
`0.2.1`.
|
|
127
127
|
|
|
128
128
|
## After production publication
|
|
129
129
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
"""EmbedFlow: progressive embedding-model migration for existing indexes."""
|
|
2
2
|
|
|
3
|
-
__version__ = "0.
|
|
3
|
+
__version__ = "0.2.0"
|
|
4
4
|
|
|
5
5
|
from .config import EmbedFlowConfig, load_config
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
def migrate(*args, **kwargs):
|
|
9
|
-
"""Start progressive migration over an existing FAISS
|
|
9
|
+
"""Start progressive migration over an existing FAISS, Qdrant, or pgvector index.
|
|
10
10
|
|
|
11
11
|
Imported lazily to keep the lightweight configuration package free of
|
|
12
12
|
model-serving dependencies at import time. See ``embedflow.migration``
|
|
@@ -129,7 +129,7 @@ def analyze_migration(
|
|
|
129
129
|
)
|
|
130
130
|
if use_registry and registry.level != "EXACT REGISTRY MATCH":
|
|
131
131
|
raise ValueError("--use-registry requires an EXACT REGISTRY MATCH")
|
|
132
|
-
engine = open_engine(config_path, device=device, demo=demo, start_worker=False)
|
|
132
|
+
engine = open_engine(config_path, device=device, demo=demo, start_worker=False, documents=docs)
|
|
133
133
|
result = run_probe(
|
|
134
134
|
engine.source_model,
|
|
135
135
|
engine.target_model,
|
|
@@ -185,6 +185,10 @@ def analyze_migration(
|
|
|
185
185
|
finally:
|
|
186
186
|
if engine is not None:
|
|
187
187
|
engine.close()
|
|
188
|
+
elif 'docs' in locals():
|
|
189
|
+
close_documents = getattr(docs, "close", None)
|
|
190
|
+
if callable(close_documents):
|
|
191
|
+
close_documents()
|
|
188
192
|
if temporary_config is not None:
|
|
189
193
|
temporary_config.unlink(missing_ok=True)
|
|
190
194
|
|