vecshift 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vecshift-0.1.0 → vecshift-0.2.0}/.github/workflows/ci.yml +11 -11
- {vecshift-0.1.0 → vecshift-0.2.0}/.github/workflows/release.yml +8 -8
- {vecshift-0.1.0 → vecshift-0.2.0}/CHANGELOG.md +16 -2
- {vecshift-0.1.0 → vecshift-0.2.0}/PKG-INFO +18 -4
- {vecshift-0.1.0 → vecshift-0.2.0}/README.md +17 -3
- {vecshift-0.1.0 → vecshift-0.2.0}/demo/run.sh +3 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/demo.md +1 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/migrations.md +41 -1
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/security.md +9 -1
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/__init__.py +1 -1
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/cli.py +46 -4
- vecshift-0.2.0/src/vecshift/cli_init.py +431 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/cli_plan.py +3 -35
- vecshift-0.2.0/src/vecshift/cli_status.py +264 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/cli_style.py +29 -0
- vecshift-0.2.0/src/vecshift/envfile.py +118 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/jobs/spec.py +39 -6
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/planning/planner.py +1 -1
- vecshift-0.2.0/tests/integration/test_pgvector_init_status.py +158 -0
- vecshift-0.2.0/tests/test_cli.py +91 -0
- vecshift-0.2.0/tests/test_envfile.py +82 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_security.py +34 -0
- vecshift-0.1.0/tests/test_cli.py +0 -36
- {vecshift-0.1.0 → vecshift-0.2.0}/.gitignore +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/Dockerfile +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/LICENSE +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/NOTICE +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/RELEASING.md +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/SECURITY.md +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/demo/compose.yaml +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/demo/out/.gitignore +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/demo/seed.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/architecture.md +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/bench.md +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/connectors/pgvector.md +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/eval.md +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/images/doctor-report.png +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/images/eval-report.png +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/images/logo/vecshift-dark.svg +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/images/logo/vecshift-light.svg +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/images/logo/vecshift-mark-dark.svg +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/images/logo/vecshift-mark-light.svg +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/images/logo/vecshift-mark.svg +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/images/social-preview.png +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/prior-art.md +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/docs/roadmap.md +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/pyproject.toml +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/scripts/release_notes.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/assets/bench.css +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/assets/eval.css +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/assets/report.css +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/assets/report.js +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/bench/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/bench/corpus.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/bench/generate.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/bench/html.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/bench/metrics.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/bench/runner.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/cli_apply.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/cli_bench.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/cli_cutover.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/cli_eval.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/connectors/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/connectors/pgvector/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/connectors/pgvector/connection.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/connectors/pgvector/documents.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/connectors/pgvector/inspect.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/connectors/pgvector/search.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/connectors/pgvector/switch.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/connectors/pgvector/target.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/connectors/pgvector/writer.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/core/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/core/capabilities.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/core/contracts.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/core/fingerprint.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/core/record.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/doctor/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/doctor/checks.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/doctor/findings.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/doctor/html.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/doctor/profile.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/embeddings/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/embeddings/cache.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/embeddings/providers.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/embeddings/spec.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/eval/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/eval/html.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/eval/metrics.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/eval/queries.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/eval/runner.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/html_kit.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/jobs/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/migrate/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/migrate/engine.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/migrate/state.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/planning/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/planning/plan.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/src/vecshift/py.typed +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/integration/__init__.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/integration/conftest.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/integration/test_pgvector_apply.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/integration/test_pgvector_bench.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/integration/test_pgvector_cutover.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/integration/test_pgvector_doctor.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/integration/test_pgvector_eval.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/integration/test_pgvector_plan.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_bench.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_contracts.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_doctor_checks.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_doctor_html.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_embeddings.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_eval.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_eval_html.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_fingerprint.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_jobs.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_migrate.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_pgvector_connection.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_planner.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_record.py +0 -0
- {vecshift-0.1.0 → vecshift-0.2.0}/tests/test_release.py +0 -0
|
@@ -25,10 +25,10 @@ jobs:
|
|
|
25
25
|
runs-on: ubuntu-latest
|
|
26
26
|
timeout-minutes: 10
|
|
27
27
|
steps:
|
|
28
|
-
- uses: actions/checkout@
|
|
28
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
29
29
|
with:
|
|
30
30
|
persist-credentials: false
|
|
31
|
-
- uses: astral-sh/setup-uv@
|
|
31
|
+
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
32
32
|
- run: uv sync --locked
|
|
33
33
|
- run: uv run ruff check .
|
|
34
34
|
- run: uv run ruff format --check .
|
|
@@ -45,10 +45,10 @@ jobs:
|
|
|
45
45
|
env:
|
|
46
46
|
UV_PYTHON: ${{ matrix.python-version }}
|
|
47
47
|
steps:
|
|
48
|
-
- uses: actions/checkout@
|
|
48
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
49
49
|
with:
|
|
50
50
|
persist-credentials: false
|
|
51
|
-
- uses: astral-sh/setup-uv@
|
|
51
|
+
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
52
52
|
- run: uv sync --locked
|
|
53
53
|
- run: uv run pytest --cov=vecshift --cov-report=term-missing
|
|
54
54
|
|
|
@@ -78,10 +78,10 @@ jobs:
|
|
|
78
78
|
env:
|
|
79
79
|
VECSHIFT_TEST_PG_DSN: postgresql://postgres:postgres@localhost:5432/postgres
|
|
80
80
|
steps:
|
|
81
|
-
- uses: actions/checkout@
|
|
81
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
82
82
|
with:
|
|
83
83
|
persist-credentials: false
|
|
84
|
-
- uses: astral-sh/setup-uv@
|
|
84
|
+
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
85
85
|
- run: uv sync --locked
|
|
86
86
|
- run: uv run pytest tests/integration
|
|
87
87
|
|
|
@@ -90,10 +90,10 @@ jobs:
|
|
|
90
90
|
runs-on: ubuntu-latest
|
|
91
91
|
timeout-minutes: 10
|
|
92
92
|
steps:
|
|
93
|
-
- uses: actions/checkout@
|
|
93
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
94
94
|
with:
|
|
95
95
|
persist-credentials: false
|
|
96
|
-
- uses: astral-sh/setup-uv@
|
|
96
|
+
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
97
97
|
- name: Known vulnerabilities in locked dependencies
|
|
98
98
|
run: |
|
|
99
99
|
uv export --frozen --all-extras --all-groups --no-hashes --no-emit-project -o requirements-audit.txt
|
|
@@ -106,10 +106,10 @@ jobs:
|
|
|
106
106
|
runs-on: ubuntu-latest
|
|
107
107
|
timeout-minutes: 10
|
|
108
108
|
steps:
|
|
109
|
-
- uses: actions/checkout@
|
|
109
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
110
110
|
with:
|
|
111
111
|
persist-credentials: false
|
|
112
|
-
- uses: astral-sh/setup-uv@
|
|
112
|
+
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
113
113
|
- run: uv build
|
|
114
114
|
- name: PyPI metadata and README render
|
|
115
115
|
run: uvx twine==7.0.0 check --strict dist/*
|
|
@@ -129,7 +129,7 @@ jobs:
|
|
|
129
129
|
PYTHON_IMAGE: mirror.gcr.io/library/python:3.13-slim@sha256:70729b46c69b4f1e97c4822c1af3df53a1476cf5ddc6c087c0c10bc3a5678c2f
|
|
130
130
|
PGVECTOR_IMAGE: mirror.gcr.io/pgvector/pgvector:pg17@sha256:ac08538c6f8b9904c33c8224c5e5706dbe760aca29db1d096972b4052c22a75d
|
|
131
131
|
steps:
|
|
132
|
-
- uses: actions/checkout@
|
|
132
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
133
133
|
with:
|
|
134
134
|
persist-credentials: false
|
|
135
135
|
- name: Run the offline demo end to end
|
|
@@ -23,10 +23,10 @@ jobs:
|
|
|
23
23
|
permissions:
|
|
24
24
|
contents: read
|
|
25
25
|
steps:
|
|
26
|
-
- uses: actions/checkout@
|
|
26
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
27
27
|
with:
|
|
28
28
|
persist-credentials: false
|
|
29
|
-
- uses: astral-sh/setup-uv@
|
|
29
|
+
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
30
30
|
with:
|
|
31
31
|
# A release never restores a cache that another workflow could have written.
|
|
32
32
|
enable-cache: false
|
|
@@ -47,7 +47,7 @@ jobs:
|
|
|
47
47
|
- name: CHANGELOG.md has notes for this version
|
|
48
48
|
run: python scripts/release_notes.py "$GITHUB_REF_NAME" > /dev/null
|
|
49
49
|
- run: uvx twine==7.0.0 check --strict dist/*
|
|
50
|
-
- uses: actions/upload-artifact@
|
|
50
|
+
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
51
51
|
with:
|
|
52
52
|
name: dist
|
|
53
53
|
path: dist/
|
|
@@ -66,7 +66,7 @@ jobs:
|
|
|
66
66
|
permissions:
|
|
67
67
|
id-token: write # trusted publishing to PyPI
|
|
68
68
|
steps:
|
|
69
|
-
- uses: actions/download-artifact@
|
|
69
|
+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
|
70
70
|
with:
|
|
71
71
|
name: dist
|
|
72
72
|
path: dist/
|
|
@@ -83,10 +83,10 @@ jobs:
|
|
|
83
83
|
id-token: write # sign the provenance attestation
|
|
84
84
|
attestations: write # store it
|
|
85
85
|
steps:
|
|
86
|
-
- uses: actions/checkout@
|
|
86
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
87
87
|
with:
|
|
88
88
|
persist-credentials: false
|
|
89
|
-
- uses: docker/setup-qemu-action@
|
|
89
|
+
- uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
|
|
90
90
|
- uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
|
91
91
|
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
|
92
92
|
with:
|
|
@@ -127,10 +127,10 @@ jobs:
|
|
|
127
127
|
permissions:
|
|
128
128
|
contents: write # create the release
|
|
129
129
|
steps:
|
|
130
|
-
- uses: actions/checkout@
|
|
130
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
131
131
|
with:
|
|
132
132
|
persist-credentials: false
|
|
133
|
-
- uses: actions/download-artifact@
|
|
133
|
+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
|
134
134
|
with:
|
|
135
135
|
name: dist
|
|
136
136
|
path: dist/
|
|
@@ -7,7 +7,20 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
-
## [0.
|
|
10
|
+
## [0.2.0] - 2026-10-10
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- `vecshift init` guides you at a terminal: it asks for the connection string (hidden),
|
|
15
|
+
lists the vector columns it finds, picks the text column, and offers a short list of
|
|
16
|
+
models and a spending limit, then shows the plan. Flags still work without prompts.
|
|
17
|
+
- Commands read settings such as `VECSHIFT_DSN` and `OPENAI_API_KEY` from a `.env` file in
|
|
18
|
+
the current folder. Variables already set take precedence; `--env-file` and
|
|
19
|
+
`--no-env-file` choose another file or none. `init` can save to it, privately.
|
|
20
|
+
- `vecshift status` shows where a migration stands and what to run next (`--json` too).
|
|
21
|
+
- Running `vecshift` alone, and `vecshift init`, show the logo at a colour terminal.
|
|
22
|
+
|
|
23
|
+
## [0.1.0] - 2026-10-10
|
|
11
24
|
|
|
12
25
|
The first release.
|
|
13
26
|
|
|
@@ -136,5 +149,6 @@ The first release.
|
|
|
136
149
|
- `Capability` flags and plugin contracts for sources, targets, and embedding providers.
|
|
137
150
|
- `vecshift fingerprint` and `vecshift --version` commands.
|
|
138
151
|
|
|
139
|
-
[Unreleased]: https://github.com/Osamamu64/vecshift/compare/v0.
|
|
152
|
+
[Unreleased]: https://github.com/Osamamu64/vecshift/compare/v0.2.0...HEAD
|
|
153
|
+
[0.2.0]: https://github.com/Osamamu64/vecshift/compare/v0.1.0...v0.2.0
|
|
140
154
|
[0.1.0]: https://github.com/Osamamu64/vecshift/releases/tag/v0.1.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: vecshift
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Zero-downtime embedding model migrations for pgvector and Supabase: plan, re-embed, evaluate, and cut over safely.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Osamamu64/vecshift
|
|
6
6
|
Project-URL: Repository, https://github.com/Osamamu64/vecshift
|
|
@@ -44,6 +44,9 @@ Description-Content-Type: text/markdown
|
|
|
44
44
|
**Safe, observable embedding migrations for any vector store, with any embedding model.**
|
|
45
45
|
|
|
46
46
|
[](https://github.com/Osamamu64/vecshift/actions/workflows/ci.yml)
|
|
47
|
+
[](https://pypi.org/project/vecshift/)
|
|
48
|
+
[](https://pypi.org/project/vecshift/)
|
|
49
|
+
[](https://github.com/Osamamu64/vecshift/pkgs/container/vecshift)
|
|
47
50
|
[](https://github.com/Osamamu64/vecshift/blob/main/LICENSE)
|
|
48
51
|

|
|
49
52
|
|
|
@@ -181,14 +184,24 @@ the [benchmarking guide](https://github.com/Osamamu64/vecshift/blob/main/docs/be
|
|
|
181
184
|
|
|
182
185
|
### Plan a migration
|
|
183
186
|
|
|
187
|
+
```bash
|
|
188
|
+
vecshift init
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
At a terminal, `init` walks you through it: paste your connection string (hidden as you
|
|
192
|
+
type), pick the vector column and the text it came from, choose the new model, and set a
|
|
193
|
+
spending limit. It writes a commented `vecshift.yaml`, can save the connection string to
|
|
194
|
+
a private `.env` file if you want it to, and shows the plan. For scripts, pass everything
|
|
195
|
+
as flags:
|
|
196
|
+
|
|
184
197
|
```bash
|
|
185
198
|
vecshift init --table public.documents --model openai/text-embedding-3-large,dims=1024
|
|
186
199
|
vecshift plan
|
|
187
200
|
```
|
|
188
201
|
|
|
189
|
-
`
|
|
190
|
-
|
|
191
|
-
|
|
202
|
+
`plan` checks the job against the database without changing anything: the SQL it would
|
|
203
|
+
run, rows, tokens, cost, duration, and storage, plus anything that would make the
|
|
204
|
+
migration fail.
|
|
192
205
|
|
|
193
206
|
### Run it
|
|
194
207
|
|
|
@@ -201,6 +214,7 @@ concurrently, while your application keeps reading and writing. It stops cleanly
|
|
|
201
214
|
Ctrl-C, at your budget, or part way with `--until 50`, and running it again resumes.
|
|
202
215
|
|
|
203
216
|
```bash
|
|
217
|
+
vecshift status # where it stands, and what to run next
|
|
204
218
|
vecshift eval # better on your data, and how fast? GO / NO-GO
|
|
205
219
|
vecshift cutover --check # safe to switch?
|
|
206
220
|
vecshift cutover # searches use the new vectors, under the same column name
|
|
@@ -8,6 +8,9 @@
|
|
|
8
8
|
**Safe, observable embedding migrations for any vector store, with any embedding model.**
|
|
9
9
|
|
|
10
10
|
[](https://github.com/Osamamu64/vecshift/actions/workflows/ci.yml)
|
|
11
|
+
[](https://pypi.org/project/vecshift/)
|
|
12
|
+
[](https://pypi.org/project/vecshift/)
|
|
13
|
+
[](https://github.com/Osamamu64/vecshift/pkgs/container/vecshift)
|
|
11
14
|
[](LICENSE)
|
|
12
15
|

|
|
13
16
|
|
|
@@ -145,14 +148,24 @@ the [benchmarking guide](docs/bench.md).
|
|
|
145
148
|
|
|
146
149
|
### Plan a migration
|
|
147
150
|
|
|
151
|
+
```bash
|
|
152
|
+
vecshift init
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
At a terminal, `init` walks you through it: paste your connection string (hidden as you
|
|
156
|
+
type), pick the vector column and the text it came from, choose the new model, and set a
|
|
157
|
+
spending limit. It writes a commented `vecshift.yaml`, can save the connection string to
|
|
158
|
+
a private `.env` file if you want it to, and shows the plan. For scripts, pass everything
|
|
159
|
+
as flags:
|
|
160
|
+
|
|
148
161
|
```bash
|
|
149
162
|
vecshift init --table public.documents --model openai/text-embedding-3-large,dims=1024
|
|
150
163
|
vecshift plan
|
|
151
164
|
```
|
|
152
165
|
|
|
153
|
-
`
|
|
154
|
-
|
|
155
|
-
|
|
166
|
+
`plan` checks the job against the database without changing anything: the SQL it would
|
|
167
|
+
run, rows, tokens, cost, duration, and storage, plus anything that would make the
|
|
168
|
+
migration fail.
|
|
156
169
|
|
|
157
170
|
### Run it
|
|
158
171
|
|
|
@@ -165,6 +178,7 @@ concurrently, while your application keeps reading and writing. It stops cleanly
|
|
|
165
178
|
Ctrl-C, at your budget, or part way with `--until 50`, and running it again resumes.
|
|
166
179
|
|
|
167
180
|
```bash
|
|
181
|
+
vecshift status # where it stands, and what to run next
|
|
168
182
|
vecshift eval # better on your data, and how fast? GO / NO-GO
|
|
169
183
|
vecshift cutover --check # safe to switch?
|
|
170
184
|
vecshift cutover # searches use the new vectors, under the same column name
|
|
@@ -55,6 +55,9 @@ fi
|
|
|
55
55
|
step "6. Switch searches to the new vectors" "vecshift cutover --yes"
|
|
56
56
|
vecshift cutover --yes
|
|
57
57
|
|
|
58
|
+
step "7. Check where it stands" "vecshift status"
|
|
59
|
+
vecshift status
|
|
60
|
+
|
|
58
61
|
printf '\nDone. Searches now use the new vectors, under the same column name.\n'
|
|
59
62
|
printf 'Reports: demo/out/doctor.html and demo/out/eval.html\n'
|
|
60
63
|
printf 'To switch back: docker compose run --rm --entrypoint vecshift vecshift rollback --yes\n'
|
|
@@ -30,6 +30,7 @@ in the vecshift container:
|
|
|
30
30
|
and writes `demo/out/eval.html`. If the verdict isn't GO, the demo stops here.
|
|
31
31
|
7. **`vecshift cutover`** renames the columns in one transaction, so `embedding` now holds
|
|
32
32
|
the new vectors.
|
|
33
|
+
8. **`vecshift status`** confirms the stage and says what to do next.
|
|
33
34
|
|
|
34
35
|
Afterwards you can switch back, or run any other command, against the same database:
|
|
35
36
|
|
|
@@ -36,7 +36,21 @@ by one model can't be compared with vectors from another.
|
|
|
36
36
|
|
|
37
37
|
## The job file
|
|
38
38
|
|
|
39
|
-
`vecshift init` writes a commented `vecshift.yaml`.
|
|
39
|
+
`vecshift init` writes a commented `vecshift.yaml`. Run it at a terminal without flags and
|
|
40
|
+
it asks for what it needs:
|
|
41
|
+
|
|
42
|
+
1. The connection string, unless `VECSHIFT_DSN` or `DATABASE_URL` is already set. It's
|
|
43
|
+
hidden as you type, checked by connecting, and asked again if it doesn't work.
|
|
44
|
+
2. The vector column, from the ones it finds, and the column holding the source text.
|
|
45
|
+
3. The model that made the current vectors (optional, for `eval`), the new model from a
|
|
46
|
+
short list or any spec, and a spending limit for paid models.
|
|
47
|
+
|
|
48
|
+
It then offers to save the connection string, and any API key the model needs, to `.env`
|
|
49
|
+
(see below), and to run `plan`. Saving defaults to no: nothing is written to disk unless
|
|
50
|
+
you say yes. With `--table` and `--model`, `init` asks nothing, for scripts; and
|
|
51
|
+
without a terminal it requires them.
|
|
52
|
+
|
|
53
|
+
The full set of settings:
|
|
40
54
|
|
|
41
55
|
```yaml
|
|
42
56
|
version: 1
|
|
@@ -68,6 +82,32 @@ The file never contains credentials: the connection string comes from the enviro
|
|
|
68
82
|
variable that `dsn_env` names. Sections whose settings are all commented out use the
|
|
69
83
|
defaults, and unknown settings are rejected so typos don't go unnoticed.
|
|
70
84
|
|
|
85
|
+
### Settings in a `.env` file
|
|
86
|
+
|
|
87
|
+
Every command reads `.env` from the current folder, if there is one, so you don't have to
|
|
88
|
+
export variables in each shell:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
VECSHIFT_DSN='postgresql://postgres.<ref>:<password>@<region>.pooler.supabase.com:5432/postgres'
|
|
92
|
+
OPENAI_API_KEY='sk-...'
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Variables already set in the environment win over the file. Use `--env-file PATH` to read
|
|
96
|
+
another file, or `--no-env-file` to read none. Keep the file private (`chmod 600 .env`;
|
|
97
|
+
vecshift warns if others can read it) and out of version control. When `init` saves to
|
|
98
|
+
it, it creates the file readable by you only, and offers to add it to `.gitignore`.
|
|
99
|
+
|
|
100
|
+
## Where a migration stands
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
vecshift status # or --json, for scripts and UIs
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`status` is read-only. It shows the stage (not started, filling the new column, ready to
|
|
107
|
+
cut over, cut over, or done), how many rows have a new vector, the index and sync
|
|
108
|
+
trigger, what `apply` has spent, the last cutover or rollback, and the command to run
|
|
109
|
+
next.
|
|
110
|
+
|
|
71
111
|
## What `plan` checks
|
|
72
112
|
|
|
73
113
|
`plan` connects read-only, samples the table, and reports what `apply` would do, what it
|
|
@@ -40,6 +40,11 @@ and how it protects credentials. To report a vulnerability, see
|
|
|
40
40
|
- Credentials inside URLs (`https://user:pass@...`) are rejected, and URL query strings
|
|
41
41
|
are hidden wherever a spec is displayed.
|
|
42
42
|
- Error messages from providers are shortened and never include the request's key.
|
|
43
|
+
- **`.env` files** are an optional alternative to exporting variables. They're read from
|
|
44
|
+
the current folder (or `--env-file`), never override variables already set, and
|
|
45
|
+
vecshift warns when other users can read one. `vecshift init` types secrets hidden, and
|
|
46
|
+
saves them to `.env` only when you say yes: the file is created `0600`, and init offers
|
|
47
|
+
to add it to `.gitignore`. Errors in the file name the line, never its value.
|
|
43
48
|
|
|
44
49
|
## The database
|
|
45
50
|
|
|
@@ -79,6 +84,7 @@ and how it protects credentials. To report a vulnerability, see
|
|
|
79
84
|
token and row counts, and the IDs of rows the provider rejected, with the provider's
|
|
80
85
|
error. Never row text or credentials. The directory is `0700` and the file `0600`, and
|
|
81
86
|
it is replaced atomically so a crash can't leave it half-written.
|
|
87
|
+
- **`.env`**: only when you choose to save to it in `vecshift init`, as above.
|
|
82
88
|
- **Files you ask for**: reports and saved queries are written only where you point them.
|
|
83
89
|
Saved generated queries contain the query text and document IDs.
|
|
84
90
|
|
|
@@ -97,7 +103,9 @@ markup, the browser would refuse to run it or send anything anywhere.
|
|
|
97
103
|
- `zizmor` for GitHub Actions security
|
|
98
104
|
- Ruff's flake8-bandit rules (`S`) for code patterns
|
|
99
105
|
- GitHub Actions are pinned to full commit SHAs, workflows get read-only tokens, and
|
|
100
|
-
checkout doesn't persist credentials. Dependabot keeps dependencies and pins up to date
|
|
106
|
+
checkout doesn't persist credentials. Dependabot keeps dependencies and pins up to date,
|
|
107
|
+
one grouped pull request per ecosystem a month, and only proposes releases at least two
|
|
108
|
+
weeks old (security updates aren't delayed).
|
|
101
109
|
- The Docker image is built from a base image pinned by digest, installs dependencies
|
|
102
110
|
from the lock file with `--require-hashes`, and runs as an unprivileged user. The demo's
|
|
103
111
|
database is reachable only from the demo's own containers.
|
|
@@ -11,17 +11,20 @@ import typer
|
|
|
11
11
|
|
|
12
12
|
from vecshift import __version__
|
|
13
13
|
from vecshift.cli_style import STYLE as _STYLE
|
|
14
|
-
from vecshift.cli_style import warn_if_password_on_command_line
|
|
14
|
+
from vecshift.cli_style import banner, warn_if_password_on_command_line
|
|
15
15
|
from vecshift.cli_style import wrap as _wrap
|
|
16
16
|
from vecshift.core.fingerprint import EmbeddingFingerprint
|
|
17
17
|
from vecshift.doctor import Report, Severity, run_checks
|
|
18
|
+
from vecshift.envfile import DEFAULT as DEFAULT_ENV_FILE
|
|
19
|
+
|
|
20
|
+
TAGLINE = "Safe, observable embedding migrations for any vector store."
|
|
18
21
|
|
|
19
22
|
app = typer.Typer(
|
|
20
23
|
name="vecshift",
|
|
21
24
|
# Tracebacks must never print local variables: they can hold connection strings and keys.
|
|
22
25
|
pretty_exceptions_show_locals=False,
|
|
23
|
-
help=
|
|
24
|
-
|
|
26
|
+
help=TAGLINE,
|
|
27
|
+
invoke_without_command=True,
|
|
25
28
|
add_completion=False,
|
|
26
29
|
)
|
|
27
30
|
|
|
@@ -32,8 +35,27 @@ def _print_version(value: bool) -> None:
|
|
|
32
35
|
raise typer.Exit()
|
|
33
36
|
|
|
34
37
|
|
|
38
|
+
def _load_env_file(path: Path | None) -> None:
|
|
39
|
+
from vecshift import envfile
|
|
40
|
+
|
|
41
|
+
if path is None:
|
|
42
|
+
return
|
|
43
|
+
try:
|
|
44
|
+
envfile.load(path)
|
|
45
|
+
except envfile.EnvFileError as exc:
|
|
46
|
+
typer.secho(f"Couldn't read {path}: {exc}", err=True, fg=typer.colors.RED)
|
|
47
|
+
raise typer.Exit(2) from exc
|
|
48
|
+
if envfile.readable_by_others(path):
|
|
49
|
+
typer.secho(
|
|
50
|
+
f"Warning: other users of this machine can read {path}. Run: chmod 600 {path}",
|
|
51
|
+
err=True,
|
|
52
|
+
fg=typer.colors.YELLOW,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
35
56
|
@app.callback()
|
|
36
57
|
def main(
|
|
58
|
+
ctx: typer.Context,
|
|
37
59
|
version: Annotated[
|
|
38
60
|
bool,
|
|
39
61
|
typer.Option(
|
|
@@ -43,8 +65,23 @@ def main(
|
|
|
43
65
|
help="Show the version and exit.",
|
|
44
66
|
),
|
|
45
67
|
] = False,
|
|
68
|
+
env_file: Annotated[
|
|
69
|
+
Path,
|
|
70
|
+
typer.Option(
|
|
71
|
+
dir_okay=False,
|
|
72
|
+
help="Read settings such as VECSHIFT_DSN from this file, if it exists. Variables "
|
|
73
|
+
"already set in the environment take precedence.",
|
|
74
|
+
),
|
|
75
|
+
] = DEFAULT_ENV_FILE,
|
|
76
|
+
no_env_file: Annotated[
|
|
77
|
+
bool, typer.Option("--no-env-file", help="Don't read a .env file.")
|
|
78
|
+
] = False,
|
|
46
79
|
) -> None:
|
|
47
80
|
"""Safe, observable embedding migrations for any vector store."""
|
|
81
|
+
_load_env_file(None if no_env_file else env_file)
|
|
82
|
+
if ctx.invoked_subcommand is None:
|
|
83
|
+
banner(__version__, TAGLINE)
|
|
84
|
+
typer.echo(ctx.get_help())
|
|
48
85
|
|
|
49
86
|
|
|
50
87
|
@app.command()
|
|
@@ -218,7 +255,8 @@ from vecshift.cli_bench import bench # noqa: E402
|
|
|
218
255
|
|
|
219
256
|
app.command()(bench)
|
|
220
257
|
|
|
221
|
-
from vecshift.
|
|
258
|
+
from vecshift.cli_init import init # noqa: E402
|
|
259
|
+
from vecshift.cli_plan import plan # noqa: E402
|
|
222
260
|
|
|
223
261
|
app.command()(init)
|
|
224
262
|
app.command()(plan)
|
|
@@ -234,3 +272,7 @@ app.command()(cleanup)
|
|
|
234
272
|
from vecshift.cli_eval import eval_ # noqa: E402
|
|
235
273
|
|
|
236
274
|
app.command(name="eval")(eval_)
|
|
275
|
+
|
|
276
|
+
from vecshift.cli_status import status # noqa: E402
|
|
277
|
+
|
|
278
|
+
app.command()(status)
|