vecshift 0.1.0__tar.gz

vecshift-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,146 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  schedule:
+    # Weekly, so newly published vulnerabilities in dependencies are caught between commits.
+    - cron: "17 6 * * 1"
+
+# Least privilege: jobs can read the repository and nothing else.
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+# Actions are pinned to full commit SHAs so a moved or compromised tag can't change what
+# runs here. Dependabot keeps the pins (and the version comments) up to date.
+
+jobs:
+  lint:
+    name: Lint and type check
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+      - run: uv sync --locked
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run mypy
+
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    env:
+      UV_PYTHON: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+      - run: uv sync --locked
+      - run: uv run pytest --cov=vecshift --cov-report=term-missing
+
+  integration:
+    name: Integration (PostgreSQL ${{ matrix.postgres }} + pgvector)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        postgres: ["16", "17"]
+    services:
+      postgres:
+        # Docker Hub's mirror on Google: the same images (identical digests), without Docker
+        # Hub's rate limit on anonymous pulls from CI runners.
+        image: mirror.gcr.io/pgvector/pgvector:pg${{ matrix.postgres }}
+        env:
+          # A throwaway database that exists only for this job.
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      VECSHIFT_TEST_PG_DSN: postgresql://postgres:postgres@localhost:5432/postgres
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+      - run: uv sync --locked
+      - run: uv run pytest tests/integration
+
+  security:
+    name: Security audit
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+      - name: Known vulnerabilities in locked dependencies
+        run: |
+          uv export --frozen --all-extras --all-groups --no-hashes --no-emit-project -o requirements-audit.txt
+          uvx pip-audit==2.10.1 --strict -r requirements-audit.txt
+      - name: GitHub Actions workflow security
+        run: uvx zizmor==1.30.1 --offline .github/workflows/
+
+  package:
+    name: Package
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+      - run: uv build
+      - name: PyPI metadata and README render
+        run: uvx twine==7.0.0 check --strict dist/*
+      - name: The wheel installs and runs on its own
+        run: |
+          uv venv /tmp/wheel-test
+          uv pip install --python /tmp/wheel-test/bin/python dist/*.whl
+          /tmp/wheel-test/bin/vecshift --version
+          /tmp/wheel-test/bin/python -c "from importlib.resources import files; assert files('vecshift').joinpath('assets/report.js').is_file()"
+
+  demo:
+    name: Docker image and demo
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      # Docker Hub's images through Google's mirror (same digests), to avoid rate limits.
+      PYTHON_IMAGE: mirror.gcr.io/library/python:3.13-slim@sha256:70729b46c69b4f1e97c4822c1af3df53a1476cf5ddc6c087c0c10bc3a5678c2f
+      PGVECTOR_IMAGE: mirror.gcr.io/pgvector/pgvector:pg17@sha256:ac08538c6f8b9904c33c8224c5e5706dbe760aca29db1d096972b4052c22a75d
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - name: Run the offline demo end to end
+        working-directory: demo
+        run: |
+          chmod a+w out
+          docker compose up --build --abort-on-container-exit --exit-code-from vecshift
+          test -s out/eval.html
+      - name: Roll back
+        working-directory: demo
+        run: docker compose run --rm --entrypoint vecshift vecshift rollback --yes
+      - if: always()
+        working-directory: demo
+        run: docker compose down --volumes

vecshift-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,146 @@
+name: Release
+
+# Pushing a tag such as v0.1.0 publishes that version: the package to PyPI, the image to
+# GitHub Container Registry, and a GitHub release with the CHANGELOG section as notes.
+# See RELEASING.md.
+
+on:
+  push:
+    tags: ["v[0-9]+.[0-9]+.[0-9]+*"]
+
+# No permissions by default; each job asks for only what it needs.
+permissions: {}
+
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Check and build
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+        with:
+          # A release never restores a cache that another workflow could have written.
+          enable-cache: false
+      - run: uv sync --locked
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run mypy
+      - run: uv run pytest
+      - run: uv build
+      - name: The tag matches the package version
+        run: |
+          version="${GITHUB_REF_NAME#v}"
+          if [ ! -f "dist/vecshift-${version}-py3-none-any.whl" ]; then
+            echo "::error::Tag ${GITHUB_REF_NAME} doesn't match the version in src/vecshift/__init__.py:"
+            ls dist
+            exit 1
+          fi
+      - name: CHANGELOG.md has notes for this version
+        run: python scripts/release_notes.py "$GITHUB_REF_NAME" > /dev/null
+      - run: uvx twine==7.0.0 check --strict dist/*
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  pypi:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    # Trusted publishing: PyPI accepts this job's OpenID Connect token, so no API token is
+    # stored anywhere. The environment can require a maintainer's approval.
+    environment:
+      name: pypi
+      url: https://pypi.org/project/vecshift/
+    permissions:
+      id-token: write # trusted publishing to PyPI
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+
+  image:
+    name: Publish the Docker image
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      packages: write # push the image to ghcr.io
+      id-token: write # sign the provenance attestation
+      attestations: write # store it
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
+      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - id: meta
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+      - id: push
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+        with:
+          context: .
+          # Docker Hub's base image through Google's mirror (same digest), to avoid rate limits.
+          build-args: |
+            PYTHON_IMAGE=mirror.gcr.io/library/python:3.13-slim@sha256:70729b46c69b4f1e97c4822c1af3df53a1476cf5ddc6c087c0c10bc3a5678c2f
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          provenance: mode=max
+          sbom: true
+      - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+        with:
+          subject-name: ghcr.io/${{ github.repository }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true
+
+  github:
+    name: GitHub release
+    needs: [pypi, image]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: write # create the release
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: dist
+          path: dist/
+      - name: Create the release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          python scripts/release_notes.py "$GITHUB_REF_NAME" > notes.md
+          prerelease=""
+          case "$GITHUB_REF_NAME" in *a*|*b*|*rc*) prerelease="--prerelease" ;; esac
+          gh release create "$GITHUB_REF_NAME" dist/* --verify-tag \
+            --repo "$GITHUB_REPOSITORY" --title "vecshift ${GITHUB_REF_NAME#v}" \
+            --notes-file notes.md $prerelease

vecshift-0.1.0/.gitignore

@@ -0,0 +1,33 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+
+# Virtual environments
+.venv/
+venv/
+
+# Tooling caches
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+coverage.xml
+htmlcov/
+
+# VecShift local state
+.vecshift/
+*.dlq.jsonl
+
+# Environment and secrets
+.env
+.env.*
+!.env.example
+
+# Editors and OS
+.idea/
+.vscode/
+.DS_Store
+requirements-audit.txt

vecshift-0.1.0/CHANGELOG.md

@@ -0,0 +1,140 @@
+# Changelog
+
+All notable changes to this project are documented here.
+
+The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project
+adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0]
+
+The first release.
+
+### Security
+
+- API keys are never sent over unencrypted `http://` to remote hosts, redirects are never
+  followed, and credentials inside model URLs are rejected. URL query strings are hidden
+  wherever a spec is shown.
+- `compat` servers only receive an API key when the spec names one with `key_env=`.
+  Previously `VECSHIFT_API_KEY` was sent to any compat URL.
+- The embedding cache is now readable by its owner only (directory `0700`, file `0600`).
+- HTML reports carry a strict Content Security Policy: no network access, and only their
+  own script, pinned by hash, may run.
+- SQL generated for `apply` quotes identifiers using the server's keyword list.
+- Crash tracebacks never print local variables, and `--dsn` with a password on the
+  command line prints a warning.
+- CI runs `pip-audit`, `zizmor`, and Ruff's security rules on every change and weekly.
+  Actions are pinned to commit SHAs, and checkout no longer persists credentials.
+
+### Fixed
+
+- On tables with constant writes, `apply` never finished (a few rows always changed during
+  each catch-up pass, so it stopped without building the index) and `cutover` could never
+  pass its final check. `apply` now finishes once at most 0.1% of rows (at least 100) are
+  left, counting them after the index build; each pass covers only what was pending when it
+  began; and `cutover` embeds up to 500 late rows while it holds the write lock.
+- `eval` treated a busy table's few waiting rows as a partial migration and skipped latency.
+- `eval`'s index recall now counts ties: a row exactly as close as the exact 10th result is
+  a correct answer. Tables with near-duplicate rows read far too low before.
+- Schema changes wait at most 2 seconds (was 5) for a lock, which bounds how long
+  application writes can queue behind them.
+- `apply` explains `could not resize shared memory segment` during the index build
+  (Docker's default 64 MB `/dev/shm`), and `plan`'s memory hint mentions `--shm-size`.
+- `plan` described cutover with the wrong column name (`_previous`; it's `_old`).
+
+- `plan` (and so `apply`) and `cutover --check` refuse a table whose row-level security binds
+  the connecting role, such as one with `FORCE ROW LEVEL SECURITY`. `apply` used to report
+  success while skipping the rows the role couldn't see.
+- They also refuse partitioned tables, where `apply` embedded every row and then failed to
+  build the index, since PostgreSQL can't build one concurrently on a partitioned table.
+
+- `vecshift doctor` sampled only the start of large tables, so rows written later (often by a
+  newer model) could be missed. Samples are now spread across the table.
+
+### Added
+
+- A Docker image that runs as a non-root user, and an offline demo:
+  `cd demo && docker compose up` runs doctor, plan, apply, eval, and cutover on a sample
+  table of English and Arabic articles, with no API key. See `docs/demo.md`.
+- Job files accept the built-in `hash/N` models, for trying vecshift out; `plan` warns
+  that they're test models (`plan.baseline_model`).
+- `vecshift eval` compares the old and new vectors on your data before cutover, without
+  re-embedding documents. It reports recall@1, recall@10, and MRR@10 overall and for each
+  query → document script pair (Arabic and Latin), how much the top results changed,
+  query embedding latency, and search latency (p50, p95, p99) with index recall across
+  `hnsw.ef_search` or `ivfflat.probes` settings. It ends with GO, NO-GO, or INCONCLUSIVE
+  (exit status 0, 1, or 3), with optional latency limits. If the old model is unavailable
+  it judges the old side from stored vectors, by how well each row's nearest rows stay in
+  the same document. Queries come from the rows themselves, a labeled file, or an LLM,
+  optionally in the other language (`--cross-language`). It's read-only, and `--dsn-env`
+  can point it at a replica.
+- `vecshift eval --html FILE` writes a self-contained report: the verdict, headline numbers
+  with their change, recall@10 per language as a before → after chart, and a latency vs
+  accuracy chart for both sides, each with a table view, in light and dark themes.
+- `source.model` in the job file names the model that made the current vectors.
+
+- `vecshift cutover` switches searches to the new vectors: in one transaction it renames
+  the live column to `<name>_old` and the new column to `<name>`, so the application's SQL
+  doesn't change. It first embeds rows added or edited since the last `apply`, then
+  confirms under a brief write lock that every row has a new vector. `--check` reports
+  whether the switch is safe without changing anything: missing vectors, an unbuilt index,
+  views bound to the column, and a vector size change the application must match.
+- `vecshift rollback` renames the columns back, and reports rows added or edited since
+  cutover that have no old-model vector.
+- `vecshift cleanup` drops the old column, its index, and its trigger once you're sure.
+- `vecshift apply --until PERCENT` stops once that share of rows has a new vector.
+
+### Changed
+
+- The sync trigger keeps a new vector the application writes itself in the same update,
+  and pins its function's `search_path`. Dropping the column it guards by hand is refused
+  instead of leaving a broken trigger behind.
+- `plan` reports a table that was already cut over and not cleaned up.
+
+- `vecshift apply` runs a migration on pgvector and Supabase: it adds the new column and a
+  trigger that clears a row's new vector when its text changes, embeds every row in
+  resumable batches, makes catch-up passes for rows edited during the run, then builds the
+  index concurrently and checks every vector's size. Writes are guarded so a vector only
+  lands if the row's text is unchanged. It stops cleanly on Ctrl-C or before passing
+  `limits.budget_usd` (counted across runs), isolates rows the provider rejects, and
+  `--json` streams progress as JSON lines. Exit status 3 means "stopped; run again".
+- `plan` now lists the sync trigger among the changes, and reports tables with a
+  composite primary key, which `apply` doesn't support yet.
+
+- `vecshift init` writes a commented `vecshift.yaml` job file describing a migration: the
+  source table, the side-by-side target column, the new model, and limits such as a budget.
+- `vecshift plan` checks a job against the database without changing anything. It shows
+  the SQL apply would run, estimates rows, tokens, cost, duration, storage, and index build
+  memory, and reports anything that would make the migration fail, exiting with status 1 on
+  errors. `--probe` measures the real model's size, token counts, and speed on 16 rows.
+- `vecshift bench` compares embedding models on a sample of your documents, from a JSONL
+  file or a pgvector table. It reports recall@1, recall@10, MRR@10, query latency,
+  throughput, cost per million documents, and storage, as a terminal table, `--json`, or an
+  `--html` leaderboard. Queries come from the documents themselves (free), from an LLM
+  (`--generate-queries`), or from your labeled file (`--queries`).
+- Embedding providers: OpenAI, Ollama, any OpenAI-compatible server, and a free hashing
+  baseline, with batching, retries that respect rate limits, known query and document
+  prefixes, and an on-disk embedding cache. Installed with `pip install 'vecshift[bench]'`.
+- `bench` shows estimated tokens and cost and asks before sending text to a remote API.
+- Project logo, in light and dark versions, with a GitHub social preview image. The HTML
+  report shows it in its header and as its browser-tab icon. `scripts/make_logo.py`
+  regenerates the logo files.
+- `vecshift doctor --html FILE` writes a self-contained HTML report: a verdict, key numbers,
+  filterable findings, and charts of vector lengths, models, and vector sizes. It works
+  offline, supports light and dark themes, and contains statistics only.
+- The JSON report gains a `facts` section with the measurements behind the findings.
+
+- `vecshift doctor` for pgvector, including Supabase: a read-only check of an existing index
+  for missing source text, mixed vector sizes or models, zero and unnormalized vectors,
+  duplicates, missing ANN indexes, row-level security gaps, and how live writes could be
+  tracked during a migration. Supports `--json` and `--fail-on` for CI.
+- PostgreSQL connections that work with every Supabase mode (direct, session pooler,
+  transaction pooler), require TLS for Supabase hosts, and never print passwords.
+- Canonical `Record` type with tombstones and `updated_at` conflict resolution.
+- `EmbeddingFingerprint` and model tags that identify a vector space.
+- `Capability` flags and plugin contracts for sources, targets, and embedding providers.
+- `vecshift fingerprint` and `vecshift --version` commands.
+
+[Unreleased]: https://github.com/Osamamu64/vecshift/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/Osamamu64/vecshift/releases/tag/v0.1.0

vecshift-0.1.0/Dockerfile

@@ -0,0 +1,35 @@
+# syntax=docker/dockerfile:1
+# The vecshift CLI in a small image that runs as a non-root user.
+#
+#   docker build -t vecshift .
+#   docker run --rm -e VECSHIFT_DSN vecshift doctor --table documents
+#
+# The base image is pinned by digest; Dependabot keeps it current. PYTHON_IMAGE can point
+# at a mirror with the same digest, such as mirror.gcr.io/library/python.
+ARG PYTHON_IMAGE=python:3.13-slim@sha256:70729b46c69b4f1e97c4822c1af3df53a1476cf5ddc6c087c0c10bc3a5678c2f
+
+FROM ${PYTHON_IMAGE} AS build
+RUN pip install --no-cache-dir uv==0.11.32
+WORKDIR /src
+COPY pyproject.toml uv.lock README.md LICENSE NOTICE ./
+COPY src ./src
+# Dependencies come from the lock file, checked against its hashes.
+RUN uv export --frozen --no-dev --no-emit-project -o /dist/requirements.txt \
+    && uv build --wheel -o /dist
+
+FROM ${PYTHON_IMAGE}
+LABEL org.opencontainers.image.title="vecshift" \
+      org.opencontainers.image.description="Zero-downtime embedding model migrations for pgvector and Supabase" \
+      org.opencontainers.image.source="https://github.com/Osamamu64/vecshift" \
+      org.opencontainers.image.licenses="Apache-2.0"
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+RUN --mount=type=bind,from=build,source=/dist,target=/dist \
+    pip install --no-cache-dir --require-hashes -r /dist/requirements.txt \
+    && pip install --no-cache-dir --no-deps /dist/*.whl \
+    && useradd --create-home --uid 1000 --user-group vecshift
+USER vecshift
+WORKDIR /home/vecshift
+ENTRYPOINT ["vecshift"]
+CMD ["--help"]