vd 0.2.1__tar.gz → 0.2.2__tar.gz

{vd-0.2.1 → vd-0.2.2}/.gitignore

@@ -113,3 +113,6 @@ venv.bak/
 
 # PyCharm
 .idea
+
+# Session handoff notes (local only)
+.claude/handoffs/

{vd-0.2.1 → vd-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vd
-Version: 0.2.1
+Version: 0.2.2
 Summary: A facade over vector databases — one interface, ~15 backends
 Project-URL: Homepage, https://github.com/i2mint/vd
 License: mit
@@ -14,6 +14,7 @@ Requires-Dist: duckdb; extra == 'all-backends'
 Requires-Dist: elasticsearch; extra == 'all-backends'
 Requires-Dist: faiss-cpu; extra == 'all-backends'
 Requires-Dist: lancedb; extra == 'all-backends'
+Requires-Dist: milvus-lite; (platform_system != 'Windows') and extra == 'all-backends'
 Requires-Dist: numpy; extra == 'all-backends'
 Requires-Dist: pgvector; extra == 'all-backends'
 Requires-Dist: pinecone; extra == 'all-backends'
@@ -63,6 +64,7 @@ Requires-Dist: numpy; extra == 'faiss'
 Provides-Extra: lancedb
 Requires-Dist: lancedb; extra == 'lancedb'
 Provides-Extra: milvus
+Requires-Dist: milvus-lite; (platform_system != 'Windows') and extra == 'milvus'
 Requires-Dist: pymilvus; extra == 'milvus'
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == 'mongodb'

{vd-0.2.1 → vd-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vd"
-version = "0.2.1"
+version = "0.2.2"
 description = "A facade over vector databases — one interface, ~15 backends"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -34,7 +34,7 @@ duckdb = ["duckdb"]
 pgvector = ["pgvector", "psycopg[binary]"]
 pinecone = ["pinecone"]
 weaviate = ["weaviate-client"]
-milvus = ["pymilvus"]
+milvus = ["pymilvus", "milvus-lite ; platform_system != 'Windows'"]
 redis = ["redis", "numpy"]
 elasticsearch = ["elasticsearch"]
 mongodb = ["pymongo"]

vd-0.2.2/tests/README.md

@@ -0,0 +1,60 @@
+# vd test suite
+
+The headline file is `test_core.py` — the **facade contract suite**. Every test
+there takes the `client` fixture, which is parametrized over every backend, so a
+single test asserts the same behaviour across all of them. That is how `vd`
+proves one uniform interface over many vector databases.
+
+## Backends the suite sweeps over
+
+`conftest.py` defines two groups:
+
+- **Embedded backends** — `memory`, `chroma`, `faiss`, `duckdb`, `lancedb`,
+  `qdrant`, `sqlite_vec`, `milvus`. No server needed; each test gets a fresh
+  client. `sqlite_vec` is skipped on a Python whose `sqlite3` lacks
+  loadable-extension support; `milvus` runs against the embedded **Milvus Lite**
+  engine and is skipped if `milvus-lite` is not installed.
+- **Server backends** — `pgvector`, `redis`, `elasticsearch`, `weaviate`,
+  `mongodb`. Each needs a running container. A backend whose port is not open is
+  **skipped**, so the suite stays green in a plain CI environment.
+
+## Running against the server backends
+
+```bash
+# Start every server backend (first run pulls images — a few minutes).
+docker compose -f tests/docker-compose.yml up -d
+
+# Wait until all five report (healthy), then run the suite.
+docker compose -f tests/docker-compose.yml ps
+
+pytest tests/
+
+# Tear down and wipe volumes when done.
+docker compose -f tests/docker-compose.yml down -v
+```
+
+Run one backend at a time with `-k`:
+
+```bash
+pytest tests/test_core.py -k pgvector
+```
+
+## Connection settings
+
+The server backends use these defaults; override with environment variables:
+
+| Backend         | Env var               | Default                                              |
+|-----------------|-----------------------|------------------------------------------------------|
+| `pgvector`      | `VD_PGVECTOR_DSN`     | `postgresql://vd:vd@localhost:5432/vd`               |
+| `redis`         | `VD_REDIS_HOST` / `VD_REDIS_PORT` | `localhost` / `6379`                     |
+| `elasticsearch` | `VD_ELASTICSEARCH_URL`| `http://localhost:9200`                              |
+| `weaviate`      | `VD_WEAVIATE_HOST`    | `localhost`                                          |
+| `mongodb`       | `VD_MONGODB_URI`      | `mongodb://localhost:27018/?directConnection=true`   |
+
+`mongodb` maps to host port **27018** (not 27017) to avoid colliding with a
+developer's native `mongod`. It uses the `mongodb-atlas-local` image because
+`$vectorSearch` requires an Atlas-capable deployment.
+
+`milvus` needs no container — it is verified against the embedded Milvus Lite
+engine, which exercises the same adapter code path as a Milvus server (only the
+client constructor differs).

vd-0.2.2/tests/conftest.py

@@ -0,0 +1,218 @@
+"""
+Pytest fixtures shared across the vd test suite.
+
+The headline fixture is :func:`client` — parametrized over every backend vd
+can reach in the current environment. A test that takes ``client`` runs once
+per backend, which is how the suite proves the facade contract holds uniformly.
+
+Two kinds of backend run:
+
+- **Embedded backends** (``memory``, ``chroma``, ``faiss``, ``duckdb``,
+  ``lancedb``, ``qdrant``, ``sqlite_vec``, ``milvus``) need no server. Each
+  test gets a fresh client. ``sqlite_vec`` is skipped on a Python whose
+  ``sqlite3`` lacks loadable-extension support; ``milvus`` runs against the
+  embedded Milvus Lite engine and is skipped if ``milvus-lite`` is absent.
+- **Server backends** (``pgvector``, ``redis``, ``elasticsearch``,
+  ``weaviate``, ``mongodb``) need a running container — see
+  ``tests/docker-compose.yml``. Each is TCP-probed and **skipped** when its
+  container is down, so the suite stays green in a plain CI environment.
+
+Connection settings for the server backends are environment-overridable
+(``VD_PGVECTOR_DSN``, ``VD_REDIS_HOST``/``VD_REDIS_PORT``,
+``VD_ELASTICSEARCH_URL``, ``VD_WEAVIATE_HOST``, ``VD_MONGODB_URI``).
+"""
+
+import hashlib
+import importlib.util
+import os
+import socket
+import sqlite3
+
+import pytest
+
+import vd
+
+#: Embedding dimension used by the test embedder.
+EMBED_DIM = 16
+
+#: Backends that need no server. Each test gets a fresh client.
+EMBEDDED_BACKENDS = [
+    "memory",
+    "chroma",
+    "faiss",
+    "duckdb",
+    "lancedb",
+    "qdrant",
+    "sqlite_vec",
+    "milvus",  # verified against the embedded Milvus Lite engine
+]
+
+#: Server backends — each needs a container (``tests/docker-compose.yml``).
+#: ``probe`` is TCP-probed; the backend is skipped when the port is closed.
+#: ``connect_kwargs`` builds the :func:`vd.connect` arguments (env-overridable).
+SERVER_BACKENDS = {
+    "pgvector": {
+        "probe": ("localhost", 5432),
+        "connect_kwargs": lambda: {
+            "dsn": os.environ.get(
+                "VD_PGVECTOR_DSN", "postgresql://vd:vd@localhost:5432/vd"
+            )
+        },
+    },
+    "redis": {
+        "probe": ("localhost", 6379),
+        "connect_kwargs": lambda: {
+            "host": os.environ.get("VD_REDIS_HOST", "localhost"),
+            "port": int(os.environ.get("VD_REDIS_PORT", "6379")),
+        },
+    },
+    "elasticsearch": {
+        "probe": ("localhost", 9200),
+        "connect_kwargs": lambda: {
+            "url": os.environ.get("VD_ELASTICSEARCH_URL", "http://localhost:9200")
+        },
+    },
+    "weaviate": {
+        "probe": ("localhost", 8080),
+        "connect_kwargs": lambda: {
+            "host": os.environ.get("VD_WEAVIATE_HOST", "localhost")
+        },
+    },
+    "mongodb": {
+        # Host port 27018 — see tests/docker-compose.yml (avoids colliding
+        # with a developer's native mongod on the default 27017).
+        "probe": ("localhost", 27018),
+        "connect_kwargs": lambda: {
+            "uri": os.environ.get(
+                "VD_MONGODB_URI", "mongodb://localhost:27018/?directConnection=true"
+            )
+        },
+    },
+}
+
+#: Every backend the parametrized ``client`` fixture sweeps over.
+ALL_BACKENDS = EMBEDDED_BACKENDS + list(SERVER_BACKENDS)
+
+
+# --------------------------------------------------------------------------- #
+# Availability probes
+# --------------------------------------------------------------------------- #
+
+
+def _tcp_open(host: str, port: int, timeout: float = 0.5) -> bool:
+    """Return ``True`` if a TCP connection to ``host:port`` succeeds."""
+    try:
+        with socket.create_connection((host, port), timeout=timeout):
+            return True
+    except OSError:
+        return False
+
+
+def _sqlite_ext_supported() -> bool:
+    """Return ``True`` if this Python's sqlite3 supports loadable extensions."""
+    try:
+        conn = sqlite3.connect(":memory:")
+        ok = hasattr(conn, "enable_load_extension")
+        conn.close()
+        return ok
+    except Exception:
+        return False
+
+
+def _unavailable_reason(name: str) -> str | None:
+    """Return a skip reason for backend ``name``, or ``None`` if it can run."""
+    if name == "sqlite_vec" and not _sqlite_ext_supported():
+        return "sqlite3 was built without loadable-extension support"
+    if name == "milvus" and importlib.util.find_spec("milvus_lite") is None:
+        return "milvus-lite not installed (embedded Milvus engine unavailable)"
+    if name in SERVER_BACKENDS:
+        host, port = SERVER_BACKENDS[name]["probe"]
+        if not _tcp_open(host, port):
+            return (
+                f"{name!r} server unreachable at {host}:{port} "
+                f"— start it with tests/docker-compose.yml"
+            )
+    return None
+
+
+def _connect_kwargs(name: str) -> dict:
+    """Return the :func:`vd.connect` kwargs for backend ``name``."""
+    if name in SERVER_BACKENDS:
+        return SERVER_BACKENDS[name]["connect_kwargs"]()
+    return {}
+
+
+def _drop_all_collections(client) -> None:
+    """
+    Delete every collection on ``client`` — best-effort.
+
+    Server backends keep state across runs; embedded backends are fresh each
+    time. Dropping everything before and after each test makes a run against a
+    live server idempotent (a re-run does not trip "already exists").
+    """
+    try:
+        names = list(client.list_collections())
+    except Exception:
+        return
+    for name in names:
+        try:
+            client.delete_collection(name)
+        except Exception:
+            pass
+
+
+# --------------------------------------------------------------------------- #
+# Test embedder
+# --------------------------------------------------------------------------- #
+
+
+def make_embedder():
+    """Return a deterministic ``text -> 16-dim vector`` embedder for tests."""
+
+    def embed(text: str) -> list[float]:
+        digest = hashlib.md5(text.encode()).digest()
+        vector = [(b / 128.0) - 1.0 for b in digest]
+        while len(vector) < EMBED_DIM:
+            vector += vector
+        return vector[:EMBED_DIM]
+
+    return embed
+
+
+@pytest.fixture
+def embedder():
+    """A deterministic test embedder."""
+    return make_embedder()
+
+
+# --------------------------------------------------------------------------- #
+# Backend fixtures
+# --------------------------------------------------------------------------- #
+
+
+@pytest.fixture(params=ALL_BACKENDS)
+def backend_name(request):
+    """Each reachable backend, one at a time; unreachable ones are skipped."""
+    name = request.param
+    if name not in vd.list_backends():
+        pytest.skip(f"backend {name!r} is not installed")
+    reason = _unavailable_reason(name)
+    if reason:
+        pytest.skip(reason)
+    return name
+
+
+@pytest.fixture
+def client(backend_name, embedder):
+    """A fresh, connected client for each backend (with an embedder)."""
+    connection = vd.connect(
+        backend_name, embedder=embedder, **_connect_kwargs(backend_name)
+    )
+    _drop_all_collections(connection)
+    yield connection
+    _drop_all_collections(connection)
+    if hasattr(connection, "close"):
+        try:
+            connection.close()
+        except Exception:
+            pass

vd-0.2.2/tests/docker-compose.yml

@@ -0,0 +1,82 @@
+# Server backends for vd's live-verification test suite.
+#
+# Brings up one container per server-backed vd backend. The pytest suite
+# (see conftest.py) TCP-probes each port and runs the contract tests against
+# whatever is reachable; backends whose container is down are skipped, so the
+# suite stays green in a plain CI environment.
+#
+#   docker compose -f tests/docker-compose.yml up -d      # start
+#   pytest tests/                                         # run the suite
+#   docker compose -f tests/docker-compose.yml down -v    # stop + wipe
+#
+# milvus is intentionally absent: the milvus adapter is verified against the
+# embedded Milvus Lite engine (no server), which exercises the same code path.
+
+services:
+  pgvector:
+    image: pgvector/pgvector:pg17
+    environment:
+      POSTGRES_USER: vd
+      POSTGRES_PASSWORD: vd
+      POSTGRES_DB: vd
+    ports:
+      - "5432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U vd"]
+      interval: 3s
+      timeout: 5s
+      retries: 20
+
+  redis:
+    image: redis/redis-stack-server:latest
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 5s
+      retries: 20
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.3
+    environment:
+      discovery.type: single-node
+      xpack.security.enabled: "false"
+      ES_JAVA_OPTS: "-Xms512m -Xmx512m"
+    ports:
+      - "9200:9200"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+
+  weaviate:
+    image: cr.weaviate.io/semitechnologies/weaviate:1.28.4
+    command: ["--host", "0.0.0.0", "--port", "8080", "--scheme", "http"]
+    environment:
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: "true"
+      PERSISTENCE_DATA_PATH: "/var/lib/weaviate"
+      DEFAULT_VECTORIZER_MODULE: "none"
+      ENABLE_MODULES: ""
+      QUERY_DEFAULTS_LIMIT: "25"
+    ports:
+      - "8080:8080"
+      - "50051:50051"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:8080/v1/.well-known/ready || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+
+  mongodb:
+    image: mongodb/mongodb-atlas-local:latest
+    # Host port 27018 (not 27017) to avoid colliding with a developer's
+    # native mongod. The container still listens on 27017 internally.
+    ports:
+      - "27018:27017"
+    healthcheck:
+      test: ["CMD-SHELL", "mongosh --quiet --eval 'db.runCommand({ ping: 1 })' || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 30

{vd-0.2.1 → vd-0.2.2}/tests/test_core.py

@@ -166,7 +166,9 @@ def test_search_with_comparison_filter(client):
     col = client.create_collection("search_cmp")
     for i in range(6):
         v = [0.0] * 16
-        v[0] = i / 6
+        # (i + 1), not i — a zero-magnitude vector has undefined cosine
+        # similarity and some backends (Elasticsearch) reject it outright.
+        v[0] = (i + 1) / 6
         col[str(i)] = _doc(str(i), v, n=i)
     hits = list(col.search([0.5] * 16, limit=10, filter={"n": {"$gte": 3}}))
     assert sorted(int(h["id"]) for h in hits) == [3, 4, 5]
@@ -176,7 +178,8 @@ def test_search_with_in_filter(client):
     col = client.create_collection("search_in")
     for i in range(5):
         v = [0.0] * 16
-        v[0] = i / 5
+        # (i + 1), not i — avoid a zero-magnitude vector (see comment above).
+        v[0] = (i + 1) / 5
         col[str(i)] = _doc(str(i), v, n=i)
     hits = list(col.search([0.5] * 16, limit=10, filter={"n": {"$in": [1, 3]}}))
     assert sorted(int(h["id"]) for h in hits) == [1, 3]
@@ -229,6 +232,8 @@ def test_upsert_is_idempotent(client):
 
 def test_dimension_mismatch_raises(client):
     col = client.create_collection("dimcheck")
-    col["ok"] = Document(id="ok", text="t", vector=[0.0] * 16)
+    # A non-zero vector — a zero-magnitude one has undefined cosine similarity
+    # and some backends (Elasticsearch) reject it outright.
+    col["ok"] = Document(id="ok", text="t", vector=[1.0] * 16)
     with pytest.raises(ValueError):
-        col["bad"] = Document(id="bad", text="t", vector=[0.0] * 8)
+        col["bad"] = Document(id="bad", text="t", vector=[1.0] * 8)

{vd-0.2.1 → vd-0.2.2}/vd/backends/elasticsearch.py

@@ -195,16 +195,28 @@ class ElasticsearchCollection(AbstractCollection):
 
     def _ensure_index(self) -> None:
         """
-        Create the Elasticsearch index on the first write if it does not exist.
-
-        The dimension must be known (set) before this is called, which is
-        guaranteed because :meth:`~vd.base.AbstractCollection._vet_vector`
-        sets ``self.dimension`` from the first vector before ``_write`` runs.
+        Ensure the ``embedding`` dense_vector field is mapped before a write.
+
+        The index itself is created eagerly by
+        :meth:`ElasticsearchClient.create_collection`, so an empty collection
+        is still visible to ``list_collections`` / ``in`` / ``get_collection``.
+        The ``dense_vector`` field, however, needs the vector dimension — not
+        known until the first vector arrives — so it is added here, lazily, via
+        ``put_mapping``. ``self.dimension`` is guaranteed set by then because
+        :meth:`~vd.base.AbstractCollection._vet_vector` runs before ``_write``.
         """
+        full_mapping = _build_mapping(self.dimension, self.metric)
         if not _index_exists(self._es, self.name):
-            self._es.indices.create(
+            # Index missing (collection created outside the adapter, or the
+            # index was dropped) — create it whole.
+            self._es.indices.create(index=self.name, mappings=full_mapping)
+            return
+        mapping = self._es.indices.get_mapping(index=self.name)
+        properties = mapping[self.name]["mappings"].get("properties", {})
+        if "embedding" not in properties:
+            self._es.indices.put_mapping(
                 index=self.name,
-                mappings=_build_mapping(self.dimension, self.metric),
+                properties={"embedding": full_mapping["properties"]["embedding"]},
             )
 
     # ----- raw primitives --------------------------------------------------- #
@@ -276,37 +288,26 @@ class ElasticsearchCollection(AbstractCollection):
         """
         Iterate all document ids in the index.
 
-        Uses a ``match_all`` query with pagination (``search_after``) to
-        retrieve ids page by page. For very large indices (tens of millions of
-        documents) prefer the Scroll API or the ``helpers.scan`` utility; the
-        ``search_after`` approach used here avoids the deprecated Scroll deep
-        pagination but still loads all ids into memory.
+        Uses the scroll-based ``helpers.scan`` to walk every document.
+        Elasticsearch disallows sorting (fielddata access) on the ``_id``
+        field, which rules out an ``_id``-keyed ``search_after`` pagination;
+        ``scan`` needs no sort. Ids are materialised into a list (as the
+        previous ``search_after`` implementation also did).
         """
         if not _index_exists(self._es, self.name):
             return iter(())
+        from elasticsearch.helpers import scan
 
-        ids: list[str] = []
-        search_after: Any = None
-        while True:
-            body: dict = {
-                "query": {"match_all": {}},
-                "size": _KEYS_PAGE_SIZE,
-                "_source": False,
-                "sort": [{"_id": "asc"}],
-            }
-            if search_after is not None:
-                body["search_after"] = search_after
-
-            response = self._es.search(index=self.name, body=body)
-            hits = response["hits"]["hits"]
-            if not hits:
-                break
-            for hit in hits:
-                ids.append(hit["_id"])
-            search_after = hits[-1]["sort"]
-            if len(hits) < _KEYS_PAGE_SIZE:
-                break
-
+        ids = [
+            hit["_id"]
+            for hit in scan(
+                self._es,
+                index=self.name,
+                query={"query": {"match_all": {}}},
+                _source=False,
+                size=_KEYS_PAGE_SIZE,
+            )
+        ]
         return iter(ids)
 
     def _count(self) -> int:
@@ -441,8 +442,12 @@ class ElasticsearchClient(AbstractClient):
         """
         Create a new collection (Elasticsearch index).
 
-        The index is created **lazily** on the first write when ``dimension``
-        is ``None``, or **eagerly** here when ``dimension`` is supplied.
+        The index is created **eagerly** so the collection is immediately
+        visible to :meth:`list_collections`, ``in``, and :meth:`get_collection`
+        even before its first write. The ``embedding`` ``dense_vector`` field
+        is added with a full mapping now when ``dimension`` is supplied, or
+        **lazily** on the first write otherwise (Elasticsearch needs the vector
+        dimension to map a ``dense_vector`` field).
 
         Parameters
         ----------
@@ -450,8 +455,8 @@ class ElasticsearchClient(AbstractClient):
             Index name. **Must be lowercase** — Elasticsearch rejects names
             with uppercase letters.
         dimension : int, optional
-            Vector dimension. Required for eager index creation; may be
-            deferred until the first write.
+            Vector dimension. When given, the ``dense_vector`` field is mapped
+            now; otherwise it is deferred until the first write.
         metric : str
             Distance metric: ``"cosine"``, ``"dot"``, or ``"l2"``.
         **index_config
@@ -472,16 +477,23 @@ class ElasticsearchClient(AbstractClient):
             dimension=dimension,
             metric=metric,
         )
-        if dimension is not None:
-            # Eager index creation: build the mapping now.
-            settings = index_config if index_config else None
-            create_kwargs: dict[str, Any] = {
-                "index": name,
-                "mappings": _build_mapping(dimension, metric),
+        # Always create the index now (an empty collection must still be
+        # listable). The dense_vector field needs the dimension, so it is only
+        # mapped here when known — otherwise _ensure_index adds it on first write.
+        mappings: dict[str, Any] = {
+            "properties": {
+                "text": {"type": "text"},
+                "metadata": {"type": "object", "enabled": False},
             }
-            if settings:
-                create_kwargs["settings"] = settings
-            self._client.indices.create(**create_kwargs)
+        }
+        if dimension is not None:
+            mappings["properties"]["embedding"] = _build_mapping(dimension, metric)[
+                "properties"
+            ]["embedding"]
+        create_kwargs: dict[str, Any] = {"index": name, "mappings": mappings}
+        if index_config:
+            create_kwargs["settings"] = index_config
+        self._client.indices.create(**create_kwargs)
         return col
 
     def get_collection(self, name: str) -> ElasticsearchCollection:

{vd-0.2.1 → vd-0.2.2}/vd/backends/milvus.py

@@ -45,6 +45,7 @@ Requires: ``pip install pymilvus``
 
 from __future__ import annotations
 
+import os
 import tempfile
 from typing import Any, Callable, Iterable, Iterator, Optional
 
@@ -514,8 +515,11 @@ class MilvusBackendClient(AbstractClient):
         elif uri is not None:
             self._client = MilvusClient(uri=uri, token=token)
         else:
-            # Milvus Lite: create a temporary .db file.
-            _fd, tmp_path = tempfile.mkstemp(suffix=".db", prefix="vd_milvus_")
+            # Milvus Lite: an embedded, server-less engine. Hand it a fresh
+            # ".db" path inside a new temp directory. milvus-lite >=3.0 treats
+            # the path as a data directory it creates itself, so it must not
+            # already exist as a file (which ``tempfile.mkstemp`` would create).
+            tmp_path = os.path.join(tempfile.mkdtemp(prefix="vd_milvus_"), "milvus.db")
             self._client = MilvusClient(tmp_path)
 
         #: Track metrics for collections created via this client so they survive