numpy-vector-store 0.3.1__tar.gz → 0.4.0__tar.gz

numpy_vector_store-0.4.0/.github/workflows/checks.yml

@@ -0,0 +1,114 @@
+name: Checks
+
+on:
+  workflow_call:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: checks-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  quality:
+    name: Lint, format, and type-check
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+        with:
+          version: "0.11.32"
+          python-version: "3.13"
+          enable-cache: true
+
+      - name: Install project dependencies
+        run: uv sync --locked --group dev
+
+      - name: Run Ruff lint checks
+        run: uv run --locked ruff check .
+
+      - name: Check Ruff formatting
+        run: uv run --locked ruff format --check .
+
+      - name: Run mypy
+        run: uv run --locked mypy src/
+
+  tests:
+    name: Test Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.11"
+          - "3.12"
+          - "3.13"
+          - "3.14"
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+        with:
+          version: "0.11.32"
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+
+      - name: Install project dependencies
+        run: uv sync --locked --group dev
+
+      - name: Run tests
+        run: uv run --locked pytest
+
+  minimum-numpy:
+    name: Test minimum NumPy on Python 3.11
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+        with:
+          version: "0.11.32"
+          python-version: "3.11"
+          enable-cache: true
+
+      - name: Run tests with the minimum NumPy version
+        run: >-
+          uv run
+          --isolated
+          --no-project
+          --python 3.11
+          --with-editable .
+          --with numpy==1.23.2
+          --with pytest==8.4.2
+          python -m pytest

{numpy_vector_store-0.3.1 → numpy_vector_store-0.4.0}/.github/workflows/publish-pypi.yml

@@ -6,8 +6,13 @@ on:
       - published
 
 jobs:
+  validate:
+    name: Validate release
+    uses: ./.github/workflows/checks.yml
+
   publish-pypi:
     name: Publish to PyPI
+    needs: validate
     runs-on: ubuntu-latest
     permissions:
       contents: read

{numpy_vector_store-0.3.1 → numpy_vector_store-0.4.0}/.github/workflows/publish-testpypi.yml

@@ -4,8 +4,13 @@ on:
   workflow_dispatch:
 
 jobs:
+  validate:
+    name: Validate release candidate
+    uses: ./.github/workflows/checks.yml
+
   publish-testpypi:
     name: Publish to TestPyPI
+    needs: validate
     runs-on: ubuntu-latest
     permissions:
       contents: read

numpy_vector_store-0.4.0/CHANGELOG.md

@@ -0,0 +1,160 @@
+# Changelog
+
+This changelog records user-visible changes to NumPy Vector Store. Earlier
+release notes remain available on the
+[GitHub releases page](https://github.com/tvanreenen/numpy-vector-store/releases).
+
+## 0.4.0 - 2026-08-09
+
+This release makes persistence explicit, self-describing, and safer to update.
+The earlier archive format stored vectors and metadata but omitted the settings
+needed to interpret those vectors correctly. Version 0.4 records that
+configuration in every new archive and introduces a lifecycle that clearly
+separates creating, opening, saving, and reloading a store.
+
+### Explicit persistence lifecycle
+
+- Add `VectorStore.open(path)` to construct a store from a versioned archive.
+  The archive supplies its own dimensions and normalization mode, so callers no
+  longer need to repeat configuration that may be wrong.
+- Let `save(path)` perform the first save or a Save As operation and bind that
+  destination. Later `save()` calls update the bound archive.
+- Add `reload()` as a deliberate refresh from disk. It always attempts to read
+  the bound archive and leaves current in-memory state unchanged if reading or
+  validation fails.
+- Keep creating a new in-memory store separate from opening one on disk. This
+  makes file access and persistence boundaries visible in application code.
+
+### Versioned, self-describing archives
+
+- Write archive format version 1 with `format_version`, `dimensions`,
+  `normalize`, `vectors`, and `metadata` fields.
+- Validate the complete archive before changing live store state, including
+  field names, scalar configuration, array dtypes and shapes, row counts,
+  finite vector values, and normalized-store zero-vector rules.
+- Reject unsupported format versions and malformed archives clearly rather
+  than inferring missing configuration or partially applying valid fields.
+- Continue preserving each metadata item as one opaque row payload.
+
+### Safer archive replacement
+
+- Write each save to a uniquely named temporary archive in the destination
+  directory, close it, and then replace the destination with `os.replace`.
+- Preserve the previous complete archive when writing or replacement fails and
+  clean up temporary files after failures.
+- Bind a new Save As destination only after its archive has been written
+  successfully.
+
+This provides an atomic visibility boundary: a reader opening the destination
+sees the previous complete archive or the new complete archive instead of a
+partially written file. It does not provide file locking, multi-writer
+coordination, or a universal power-loss durability guarantee.
+
+### Short migration window
+
+- Keep constructor `file_path=`, instance `load()`, and direct context-manager
+  persistence for version 0.4 with `FutureWarning`. They will be removed in
+  0.5.
+- Make the deprecated context manager save only after a successful block. If
+  the block raises, it does not save or suppress the exception.
+- Keep a configuration-aware reader for older archives containing only
+  `vectors` and `metadata`. Loading one warns, and its next save rewrites it as
+  format version 1.
+- Intentionally make `open()` reject an unversioned archive because that file
+  cannot report its original dimensions or normalization semantics.
+- Add a dedicated [persistence migration guide](MIGRATION.md) with side-by-side
+  API replacements and one-time legacy archive conversion instructions.
+
+The legacy API and unversioned archive reader are removed in 0.5. Applications
+should migrate an old archive once with 0.4 or recreate it from source data;
+indefinite compatibility with the incomplete two-array format is not planned.
+
+### Runtime compatibility
+
+- Support Python 3.11 through 3.14. Python 3.10 remains supported by the 0.3
+  release series but is not supported by 0.4.
+- Raise the minimum NumPy version from 1.21.3 to 1.23.2, the earliest release
+  that supports Python 3.11.
+- Exercise every supported Python version in CI and test NumPy 1.23.2 in a
+  dedicated minimum-dependency job.
+
+### Upgrade notes and boundaries
+
+- Search, insertion, retrieval, clearing, normalization, and metadata behavior
+  are unchanged from 0.3.2.
+- Code using `VectorStore.open()`, `save(path)`, `save()`, and `reload()` is on
+  the persistence API intended for 0.5.
+- Code using a transitional entry point continues to work in 0.4 but emits a
+  warning so the required 0.5 migration is visible during testing.
+- Metadata still uses NumPy's pickle-backed object-array loading. Only open
+  archives produced by your application or another trusted source.
+- Mutable public state, repeated-add performance, deterministic tie ordering,
+  and a formal thread-safety contract remain planned for later releases.
+
+## 0.3.2 - 2026-07-27
+
+This reliability and performance patch makes existing vector storage, search,
+metadata, and persistence behavior safer and more predictable. It does not
+intentionally break valid existing usage or change the `.npz` archive format.
+
+### Numerical reliability
+
+- Reject vectors, queries, and search thresholds that contain non-finite values
+  or cannot remain finite when represented as `float32`. Invalid input now
+  fails before it can corrupt stored state or ranking.
+- Calculate norms and raw metric intermediates with `float64` where `float32`
+  could overflow or underflow. Large and very small finite vectors can now be
+  normalized and compared reliably.
+- Allow zero vectors in stores created with `normalize=False`, where they are
+  valid for dot-product and Euclidean search.
+- Raise a clear error if a raw cosine search includes a zero vector, because
+  cosine similarity is undefined for that row.
+- Avoid duplicate full-size `float64` buffers when calculating raw Euclidean
+  distance.
+
+### Persistence
+
+- Resolve a path without an `.npz` suffix to the same archive for both saving
+  and loading. For example, `file_path="vectors"` consistently uses
+  `vectors.npz`.
+- Allow `load()` to be retried when the persistence file did not exist during
+  an earlier attempt.
+- Reset load state in `clear()` so a subsequent explicit `load()` can restore
+  the saved rows.
+- Keep repeated `load()` calls idempotent after a successful load.
+
+### Metadata
+
+- Preserve each item in the outer metadata sequence as one opaque row payload.
+  Tuples and lists are no longer mistaken for extra NumPy array dimensions.
+- Support dictionary, dataclass, tuple, list, string, integer, and other scalar
+  payloads consistently through insertion, search results, saving, and loading.
+- Continue rejecting explicitly multidimensional NumPy metadata arrays rather
+  than silently flattening ambiguous input.
+
+### Search memory use
+
+- Search the stored vector matrix directly when `within_rows` is omitted,
+  avoiding an unnecessary full-matrix copy on every unfiltered query.
+- Preserve original store indexes and metadata when `within_rows` selects a
+  filtered subset.
+- Document that filtered searches allocate a temporary matrix proportional to
+  the selected row count and vector dimensions.
+
+### Compatibility and validation
+
+- Test Python 3.10 through 3.14 in GitHub Actions.
+- Test the minimum supported NumPy version in a dedicated Python 3.10 job.
+- Raise the minimum NumPy requirement from 1.20 to 1.21.3 so it is compatible
+  with the oldest supported Python version.
+- Require linting, formatting, type checking, and the full Python test matrix
+  before publishing to PyPI.
+
+### Upgrade notes
+
+- No public method signatures or persisted field names changed.
+- Existing trusted `.npz` archives with `vectors` and `metadata` remain
+  readable.
+- Environments using NumPy 1.20 must upgrade to NumPy 1.21.3 or newer.
+- Inputs that previously produced `nan`, `inf`, or unreliable rankings now
+  raise `ValueError` instead.

numpy_vector_store-0.4.0/MIGRATION.md

@@ -0,0 +1,169 @@
+# Persistence migration guide
+
+Version 0.4 introduces a persistence lifecycle that separates creating a new
+store, opening an existing archive, saving, and deliberately refreshing from
+disk. It also provides a one-release bridge for applications and archives using
+the earlier API.
+
+The transition is intentionally short. Version 0.4 emits `FutureWarning` for
+constructor `file_path=`, instance `load()`, direct context-manager use, and
+unversioned archives. Version 0.5 removes those entry points and the
+unversioned archive reader.
+
+## Creating and saving a new store
+
+Previously, the destination was supplied while constructing the store:
+
+```python
+store = VectorStore(dimensions=1536, file_path="vectors.npz")
+store.add(vectors, metadata)
+store.save()
+```
+
+Create the in-memory store first, then bind its destination with the first
+save:
+
+```python
+store = VectorStore(dimensions=1536)
+store.add(vectors, metadata)
+store.save("vectors.npz")
+```
+
+Later `save()` calls update the bound archive. Supplying another path performs
+a Save As operation and binds the new destination after the write succeeds.
+Calling `save()` before a store is bound raises `ValueError` rather than
+silently leaving the data unsaved.
+
+## Opening an existing archive
+
+The old API required callers to repeat configuration that should belong to the
+archive:
+
+```python
+store = VectorStore(
+    dimensions=1536,
+    file_path="vectors.npz",
+    normalize=True,
+)
+store.load()
+```
+
+Open a version 1 archive directly:
+
+```python
+store = VectorStore.open("vectors.npz")
+```
+
+`open()` restores `dimensions` and `normalize` from the archive, validates its
+contents, loads its rows, and binds its path. Applications no longer need to
+keep archive configuration separately or risk loading the same vectors with
+different semantics.
+
+The generic parameter still describes application metadata. It can be kept
+when useful:
+
+```python
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ChunkMetadata:
+    source: str
+    chunk_index: int
+
+
+store = VectorStore[ChunkMetadata].open("vectors.npz")
+```
+
+`ChunkMetadata` is an example application type, not a class provided by this
+library.
+
+## Refreshing from disk
+
+Use `reload()` when another process may have changed the bound archive and the
+current in-memory changes should be discarded:
+
+```python
+store = VectorStore.open("vectors.npz")
+
+# Later, after the file may have changed:
+store.reload()
+```
+
+Unlike transitional `load()`, `reload()` always attempts to read. It raises if
+the store is unbound, the file is missing, or the archive is invalid. A failed
+reload leaves the current in-memory vectors and metadata unchanged.
+
+## Replacing context-manager persistence
+
+The earlier context manager saved automatically on exit:
+
+```python
+with VectorStore(dimensions=1536, file_path="vectors.npz") as store:
+    store.add(vectors, metadata)
+```
+
+Use an explicit save after the work succeeds:
+
+```python
+store = VectorStore(dimensions=1536)
+store.add(vectors, metadata)
+store.save("vectors.npz")
+```
+
+Normal Python control flow already prevents the final line from running if
+`add()` raises. The persistence boundary is visible, and readers do not need to
+remember an implicit exit side effect.
+
+During 0.4, the deprecated context manager saves only after normal completion.
+It does not save while an exception is propagating and does not suppress the
+exception. There is no planned replacement autosave context manager.
+
+## Migrating an archive created before 0.4
+
+Older archives contain only `vectors` and `metadata`. They do not record their
+dimensions or whether vectors use normalized or raw semantics, so `open()`
+cannot construct a correct store from them.
+
+Use the 0.4 compatibility API once with the archive's original configuration:
+
+```python
+legacy = VectorStore(
+    dimensions=1536,
+    file_path="legacy-vectors.npz",
+    normalize=True,
+)
+legacy.load()
+legacy.save()
+```
+
+This code emits transition warnings by design. The final `save()` rewrites the
+archive as format version 1 with `format_version`, `dimensions`, `normalize`,
+`vectors`, and `metadata`. It can then use the preferred API:
+
+```python
+store = VectorStore.open("legacy-vectors.npz")
+```
+
+Applications that can recreate archives from source vectors and metadata may
+choose to do that instead. The unversioned reader is removed in 0.5 rather than
+maintained as a long-term compatibility format.
+
+## Removal schedule
+
+| Transitional behavior | 0.4 | 0.5 |
+|---|---|---|
+| Constructor `file_path=` | Works with `FutureWarning` | Removed |
+| Instance `load()` | Works with `FutureWarning` | Removed |
+| Direct context-manager persistence | Saves only on successful exit and warns | Removed |
+| Unversioned two-array archives | Load with known configuration and warn | Reader removed |
+| `open()`, `save(path)`, `save()`, and `reload()` | Preferred | Supported |
+
+## Persistence boundaries that do not change
+
+Metadata is stored in a pickle-backed NumPy object array. Archives remain
+trusted input and must not be opened from untrusted or unverifiable sources.
+
+Saves use same-directory temporary files and atomic replacement, but the
+library does not add file locking, coordinate concurrent writers, or promise
+power-loss durability across every operating system and filesystem.