polign 0.1.0__tar.gz

polign-0.1.0/.gitignore

@@ -0,0 +1,54 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool
+*.out
+
+# Dependency directories
+vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# Build output (incl. `go build ./cmd/...` binaries dropped at the repo root)
+/bin/
+/dist/
+/ann
+/apikey
+/ivfpq-compact
+/loadtest
+/maintain
+/persistor
+/polign
+/server
+example/demo/demo
+
+# Benchmark run logs
+bench/**/*.log
+
+# Native install data dir (default POLIGN_DATA_DIR)
+/.polign-data/
+
+# Environment / secrets
+.env
+.env.native
+
+# Internal design docs (kept local, not published)
+/docs/
+
+# macOS Finder metadata
+.DS_Store
+
+# Python SDK (sdk/python)
+__pycache__/
+*.egg-info/
+sdk/python/dist/
+sdk/python/.pytest_cache/

polign-0.1.0/PKG-INFO

@@ -0,0 +1,194 @@
+Metadata-Version: 2.4
+Name: polign
+Version: 0.1.0
+Summary: Python client for polign_db, a vector database with hot/cold search and hybrid BM25 fusion
+Project-URL: Homepage, https://polign.com
+Project-URL: Documentation, https://polign.com/python.html
+Project-URL: Issues, https://github.com/Polign/polign/issues
+Author: Polign
+License: Apache-2.0
+Keywords: embeddings,polign,similarity search,vector database,vector search
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Provides-Extra: grpc
+Requires-Dist: grpcio>=1.80; extra == 'grpc'
+Requires-Dist: protobuf<7,>=6.31; extra == 'grpc'
+Description-Content-Type: text/markdown
+
+# polign — Python client for polign_db
+
+A thin Python client for [polign](https://polign.com) with
+two interchangeable transports:
+
+- **HTTP** (`polign.Client`) — zero dependencies, pure stdlib. Talks JSON to
+  the server's HTTP listener (default `:23000`).
+- **gRPC** (`polign.GrpcClient`) — install with the `[grpc]` extra.
+  Wire-compatible with the Go client, against the gRPC listener (default
+  `:23001`).
+
+Both expose the same seven operations with identical semantics: `put`,
+`put_many`, `get`, `get_many`, `list`, `delete`, `search`.
+
+## Install
+
+Not yet published to PyPI — install from a checkout of the repo:
+
+```bash
+pip install ./sdk/python           # HTTP client, no dependencies
+pip install './sdk/python[grpc]'   # + gRPC transport (grpcio, protobuf)
+```
+
+(From inside this directory: `pip install .`; add `-e` for an editable
+development install — needs pip ≥ 21.3.)
+
+## Quick start
+
+```python
+from polign import Client
+
+client = Client("http://localhost:23000")
+
+# Upsert. Collections are auto-created on first put, inferring their
+# dimension from the vector. Values accept lists or numpy arrays.
+client.put("docs", "doc-1", embedding, metadata={"title": "Cats", "url": "/cats"})
+
+# Nearest-neighbour search (distance: smaller = closer)
+for hit in client.search("docs", values=query_embedding, k=10):
+    print(hit.id, hit.distance, hit.metadata)
+```
+
+Swap in gRPC by changing two lines — the rest of the code is identical:
+
+```python
+from polign import GrpcClient
+
+client = GrpcClient("localhost:23001")
+```
+
+## Operations
+
+```python
+client.put("docs", "doc-1", values, metadata={"k": "v"})  # upsert, returns id
+client.put_many("docs", [Vector(id="a", values=va), Vector(id="b", values=vb)])
+                                               # batch upsert, one request
+v = client.get("docs", "doc-1")                # Vector(id, values, metadata)
+vs = client.get_many("docs", ["a", "b"])       # batch read, byte-exact values:
+                                               # never a compressed reconstruction
+                                               # (get may return one on a cold-
+                                               # flushed collection); unknown ids
+                                               # omitted, request order kept
+page = client.list("docs", limit=100, offset=0)  # page.vectors, page.total
+client.delete("docs", "doc-1")                 # True; False if absent (no error)
+hits = client.search("docs", values=q, k=10)   # [Hit(id, distance, score, metadata)]
+```
+
+### Search options
+
+```python
+from polign import Fusion
+
+client.search(
+    "docs",
+    values=q,                      # vector leg (either values or text required)
+    k=10,
+    ef=64,                         # HNSW beam width override (0 = server default)
+    filter={"lang": "en"},         # metadata predicate (see below)
+    text="quick brown fox",        # BM25 leg (needs a segment index server-side)
+    fusion=Fusion(method="linear", alpha=0.6),  # hybrid fusion; default RRF
+    cold=True, nprobe=8,           # serve from object-store segments
+)
+```
+
+`text` alone runs a pure BM25 search; `values` + `text` runs hybrid search
+fused server-side. `hit.score` is the BM25/fused relevance (larger = better)
+and is `0.0` on a pure vector search.
+
+`filter` takes the same dict language on both transports (see
+`docs/FILTERING.md` in the main repo): bare values are equality
+(ANDed across keys); per-key operator objects (`$eq`, `$ne`, `$in`, `$gt`,
+`$gte`, `$lt`, `$lte`, `$exists`) and the composers `$and`/`$or`/`$not`
+express richer predicates:
+
+```python
+filter={
+    "tenant": "acme",
+    "score": {"$gte": 0.5},
+    "$or": [{"lang": "en"}, {"lang": {"$exists": False}}],
+}
+```
+
+## Auth and tenants
+
+```python
+client = Client(
+    "https://db.example.com:23000",
+    api_key="plgn_<key_id>_<secret>",     # sent as Authorization: Bearer
+    tenant="acme/search/prod",            # org/project/namespace
+)
+```
+
+Servers started without `-auth-stores` need no credentials. With TLS enabled
+server-side, use an `https://` URL (HTTP) or pass
+`credentials=grpc.ssl_channel_credentials()` (gRPC).
+
+## Errors
+
+All errors subclass `polign.PolignError`:
+
+| Exception               | HTTP | gRPC                 |
+|-------------------------|------|----------------------|
+| `InvalidArgumentError`  | 400  | `INVALID_ARGUMENT`   |
+| `AuthenticationError`   | 401  | `UNAUTHENTICATED`    |
+| `PermissionDeniedError` | 403  | `PERMISSION_DENIED`  |
+| `NotFoundError`         | 404  | `NOT_FOUND`          |
+| `NotOwnerError`         | 421  | `FAILED_PRECONDITION`|
+| `RateLimitError`        | 429  | `RESOURCE_EXHAUSTED` |
+| `ServerError`           | 5xx  | `INTERNAL`           |
+| `ConnectionError`       | —    | `UNAVAILABLE`        |
+
+`NotOwnerError.owner` names the owning node in fleet mode — reconnect there
+and retry.
+
+## Notes & caveats
+
+Mirrors the Go client's caveats (see `docs/CLIENT.md` in the main repo):
+
+- Embed documents and queries with the **same model** — distances are only
+  meaningful within one embedding space.
+- Auto-created collections use the server's default metric (L2) and hybrid
+  IVF index; metric and index tuning are not yet exposed over the wire.
+- **Bulk loads should use `put_many`** — one request per batch instead of one
+  per vector. The server validates the whole batch up front (id, non-empty
+  values, uniform dimension, at most 5000 vectors per batch: an invalid batch
+  applies nothing); on a rarer mid-batch failure earlier vectors remain
+  applied, and since puts are idempotent upserts you simply retry the batch.
+  Chunk larger loads into batches of 5000.
+- Metadata is `str -> str` only; filter scalars (numbers, booleans) compare
+  against the stored string by their literal form (`"0.5"`, `"true"`).
+
+## Development
+
+```bash
+cd sdk/python
+pip install -e .[grpc] pytest
+pytest tests/test_unit.py          # stub-server unit tests
+pytest tests/test_integration.py   # builds & boots the real server (needs Go)
+```
+
+Regenerate the vendored gRPC stubs after changing `proto/vectordb.proto`
+(from the repo root):
+
+```bash
+python -m grpc_tools.protoc -I proto \
+    --python_out=sdk/python/polign/_pb \
+    --grpc_python_out=sdk/python/polign/_pb \
+    --pyi_out=sdk/python/polign/_pb \
+    proto/vectordb.proto
+sed -i '' 's/^import vectordb_pb2 as/from . import vectordb_pb2 as/' \
+    sdk/python/polign/_pb/vectordb_pb2_grpc.py
+```

polign-0.1.0/README.md

@@ -0,0 +1,172 @@
+# polign — Python client for polign_db
+
+A thin Python client for [polign](https://polign.com) with
+two interchangeable transports:
+
+- **HTTP** (`polign.Client`) — zero dependencies, pure stdlib. Talks JSON to
+  the server's HTTP listener (default `:23000`).
+- **gRPC** (`polign.GrpcClient`) — install with the `[grpc]` extra.
+  Wire-compatible with the Go client, against the gRPC listener (default
+  `:23001`).
+
+Both expose the same seven operations with identical semantics: `put`,
+`put_many`, `get`, `get_many`, `list`, `delete`, `search`.
+
+## Install
+
+Not yet published to PyPI — install from a checkout of the repo:
+
+```bash
+pip install ./sdk/python           # HTTP client, no dependencies
+pip install './sdk/python[grpc]'   # + gRPC transport (grpcio, protobuf)
+```
+
+(From inside this directory: `pip install .`; add `-e` for an editable
+development install — needs pip ≥ 21.3.)
+
+## Quick start
+
+```python
+from polign import Client
+
+client = Client("http://localhost:23000")
+
+# Upsert. Collections are auto-created on first put, inferring their
+# dimension from the vector. Values accept lists or numpy arrays.
+client.put("docs", "doc-1", embedding, metadata={"title": "Cats", "url": "/cats"})
+
+# Nearest-neighbour search (distance: smaller = closer)
+for hit in client.search("docs", values=query_embedding, k=10):
+    print(hit.id, hit.distance, hit.metadata)
+```
+
+Swap in gRPC by changing two lines — the rest of the code is identical:
+
+```python
+from polign import GrpcClient
+
+client = GrpcClient("localhost:23001")
+```
+
+## Operations
+
+```python
+client.put("docs", "doc-1", values, metadata={"k": "v"})  # upsert, returns id
+client.put_many("docs", [Vector(id="a", values=va), Vector(id="b", values=vb)])
+                                               # batch upsert, one request
+v = client.get("docs", "doc-1")                # Vector(id, values, metadata)
+vs = client.get_many("docs", ["a", "b"])       # batch read, byte-exact values:
+                                               # never a compressed reconstruction
+                                               # (get may return one on a cold-
+                                               # flushed collection); unknown ids
+                                               # omitted, request order kept
+page = client.list("docs", limit=100, offset=0)  # page.vectors, page.total
+client.delete("docs", "doc-1")                 # True; False if absent (no error)
+hits = client.search("docs", values=q, k=10)   # [Hit(id, distance, score, metadata)]
+```
+
+### Search options
+
+```python
+from polign import Fusion
+
+client.search(
+    "docs",
+    values=q,                      # vector leg (either values or text required)
+    k=10,
+    ef=64,                         # HNSW beam width override (0 = server default)
+    filter={"lang": "en"},         # metadata predicate (see below)
+    text="quick brown fox",        # BM25 leg (needs a segment index server-side)
+    fusion=Fusion(method="linear", alpha=0.6),  # hybrid fusion; default RRF
+    cold=True, nprobe=8,           # serve from object-store segments
+)
+```
+
+`text` alone runs a pure BM25 search; `values` + `text` runs hybrid search
+fused server-side. `hit.score` is the BM25/fused relevance (larger = better)
+and is `0.0` on a pure vector search.
+
+`filter` takes the same dict language on both transports (see
+`docs/FILTERING.md` in the main repo): bare values are equality
+(ANDed across keys); per-key operator objects (`$eq`, `$ne`, `$in`, `$gt`,
+`$gte`, `$lt`, `$lte`, `$exists`) and the composers `$and`/`$or`/`$not`
+express richer predicates:
+
+```python
+filter={
+    "tenant": "acme",
+    "score": {"$gte": 0.5},
+    "$or": [{"lang": "en"}, {"lang": {"$exists": False}}],
+}
+```
+
+## Auth and tenants
+
+```python
+client = Client(
+    "https://db.example.com:23000",
+    api_key="plgn_<key_id>_<secret>",     # sent as Authorization: Bearer
+    tenant="acme/search/prod",            # org/project/namespace
+)
+```
+
+Servers started without `-auth-stores` need no credentials. With TLS enabled
+server-side, use an `https://` URL (HTTP) or pass
+`credentials=grpc.ssl_channel_credentials()` (gRPC).
+
+## Errors
+
+All errors subclass `polign.PolignError`:
+
+| Exception               | HTTP | gRPC                 |
+|-------------------------|------|----------------------|
+| `InvalidArgumentError`  | 400  | `INVALID_ARGUMENT`   |
+| `AuthenticationError`   | 401  | `UNAUTHENTICATED`    |
+| `PermissionDeniedError` | 403  | `PERMISSION_DENIED`  |
+| `NotFoundError`         | 404  | `NOT_FOUND`          |
+| `NotOwnerError`         | 421  | `FAILED_PRECONDITION`|
+| `RateLimitError`        | 429  | `RESOURCE_EXHAUSTED` |
+| `ServerError`           | 5xx  | `INTERNAL`           |
+| `ConnectionError`       | —    | `UNAVAILABLE`        |
+
+`NotOwnerError.owner` names the owning node in fleet mode — reconnect there
+and retry.
+
+## Notes & caveats
+
+Mirrors the Go client's caveats (see `docs/CLIENT.md` in the main repo):
+
+- Embed documents and queries with the **same model** — distances are only
+  meaningful within one embedding space.
+- Auto-created collections use the server's default metric (L2) and hybrid
+  IVF index; metric and index tuning are not yet exposed over the wire.
+- **Bulk loads should use `put_many`** — one request per batch instead of one
+  per vector. The server validates the whole batch up front (id, non-empty
+  values, uniform dimension, at most 5000 vectors per batch: an invalid batch
+  applies nothing); on a rarer mid-batch failure earlier vectors remain
+  applied, and since puts are idempotent upserts you simply retry the batch.
+  Chunk larger loads into batches of 5000.
+- Metadata is `str -> str` only; filter scalars (numbers, booleans) compare
+  against the stored string by their literal form (`"0.5"`, `"true"`).
+
+## Development
+
+```bash
+cd sdk/python
+pip install -e .[grpc] pytest
+pytest tests/test_unit.py          # stub-server unit tests
+pytest tests/test_integration.py   # builds & boots the real server (needs Go)
+```
+
+Regenerate the vendored gRPC stubs after changing `proto/vectordb.proto`
+(from the repo root):
+
+```bash
+python -m grpc_tools.protoc -I proto \
+    --python_out=sdk/python/polign/_pb \
+    --grpc_python_out=sdk/python/polign/_pb \
+    --pyi_out=sdk/python/polign/_pb \
+    proto/vectordb.proto
+sed -i '' 's/^import vectordb_pb2 as/from . import vectordb_pb2 as/' \
+    sdk/python/polign/_pb/vectordb_pb2_grpc.py
+```

polign-0.1.0/polign/__init__.py

@@ -0,0 +1,66 @@
+"""polign — Python client for polign_db.
+
+HTTP client (zero dependencies):
+
+    from polign import Client
+    c = Client("http://localhost:23000")
+
+gRPC client (pip install polign[grpc]):
+
+    from polign import GrpcClient
+    c = GrpcClient("localhost:23001")
+
+Both expose the same operations: put, put_many, get, list, delete, search.
+"""
+
+from .client import Client
+from .errors import (
+    AuthenticationError,
+    ConflictError,
+    ConnectionError,
+    InvalidArgumentError,
+    NotEnabledError,
+    NotFoundError,
+    NotOwnerError,
+    PermissionDeniedError,
+    PolignError,
+    RateLimitError,
+    ServerError,
+    UnavailableError,
+)
+from .types import CollectionBackend, CollectionInfo, Fusion, Hit, Vector, VectorPage
+
+__version__ = "0.1.0rc8"
+
+__all__ = [
+    "Client",
+    "GrpcClient",
+    "Vector",
+    "Hit",
+    "Fusion",
+    "VectorPage",
+    "PolignError",
+    "ConnectionError",
+    "InvalidArgumentError",
+    "NotFoundError",
+    "AuthenticationError",
+    "PermissionDeniedError",
+    "RateLimitError",
+    "NotOwnerError",
+    "ServerError",
+    "ConflictError",
+    "NotEnabledError",
+    "UnavailableError",
+    "CollectionBackend",
+    "CollectionInfo",
+    "__version__",
+]
+
+
+def __getattr__(name):
+    # Lazy import so the base package works without grpcio installed.
+    if name == "GrpcClient":
+        from .grpc_client import GrpcClient
+
+        return GrpcClient
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

polign-0.1.0/polign/_filter.py

@@ -0,0 +1,161 @@
+"""Converts the dict filter language into the wire FilterExpr tree.
+
+The gRPC client accepts the same metadata-filter dicts as the HTTP client
+(the syntax of docs/FILTERING.md) and converts them client-side into the
+``polign.v1.FilterExpr`` proto. The semantics mirror the server's JSON parser
+(internal/filter/json.go): bare values are equality, per-key operator objects
+($eq, $ne, $in, $gt, $gte, $lt, $lte, $exists) AND together with the four
+range operators merged into one range, and $and/$or/$not compose sub-filters.
+
+Only imported by the gRPC transport — needs the ``grpc`` extra for protobuf.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from . import errors
+from ._pb import vectordb_pb2 as pb
+
+_RANGE_OPS = {"$gt": "gt", "$gte": "gte", "$lt": "lt", "$lte": "lte"}
+
+
+def filter_expr_from_dict(obj: Optional[Dict[str, Any]]) -> Optional["pb.FilterExpr"]:
+    """Convert a filter dict to a FilterExpr, or None for no filter.
+
+    Raises :class:`polign.InvalidArgumentError` on a malformed filter, with
+    the same messages the server would produce.
+    """
+    if not obj:
+        return None
+    if not isinstance(obj, dict):
+        raise errors.InvalidArgumentError("filter: not a JSON object")
+    return _parse_object(obj)
+
+
+def _parse_object(obj: Dict[str, Any]) -> Optional["pb.FilterExpr"]:
+    exprs: List[pb.FilterExpr] = []
+    for key in sorted(obj):
+        val = obj[key]
+        if key in ("$and", "$or"):
+            children = _parse_object_list(key, val)
+            junction = pb.FilterJunction(exprs=children)
+            if key == "$and":
+                exprs.append(pb.FilterExpr(**{"and": junction}))
+            else:
+                exprs.append(pb.FilterExpr(**{"or": junction}))
+        elif key == "$not":
+            if not isinstance(val, dict):
+                raise errors.InvalidArgumentError("filter: $not takes a filter object")
+            child = _parse_object(val)
+            if child is None:
+                raise errors.InvalidArgumentError(
+                    "filter: $not takes a non-empty filter object"
+                )
+            exprs.append(pb.FilterExpr(**{"not": child}))
+        elif key.startswith("$"):
+            raise errors.InvalidArgumentError(f"filter: unknown operator {key!r}")
+        else:
+            exprs.append(_parse_field(key, val))
+    return _and_collapse(exprs)
+
+
+def _parse_object_list(op: str, val: Any) -> List["pb.FilterExpr"]:
+    if not isinstance(val, list) or not val:
+        raise errors.InvalidArgumentError(
+            f"filter: {op} takes a non-empty array of filter objects"
+        )
+    children = []
+    for item in val:
+        if not isinstance(item, dict):
+            raise errors.InvalidArgumentError(
+                f"filter: {op} takes a non-empty array of filter objects"
+            )
+        child = _parse_object(item)
+        if child is None:
+            raise errors.InvalidArgumentError(f"filter: {op}: empty filter object")
+        children.append(child)
+    return children
+
+
+def _parse_field(key: str, val: Any) -> "pb.FilterExpr":
+    if isinstance(val, dict):
+        return _parse_field_ops(key, val)
+    return _cond(pb.FilterCond(key=key, eq=_scalar(key, val)))
+
+
+def _parse_field_ops(key: str, ops: Dict[str, Any]) -> "pb.FilterExpr":
+    if not ops:
+        raise errors.InvalidArgumentError(f"filter: key {key!r}: empty operator object")
+    exprs: List[pb.FilterExpr] = []
+    bounds: Dict[str, str] = {}
+    numeric_set = lex_set = False
+    for op in sorted(ops):
+        val = ops[op]
+        if op == "$eq":
+            exprs.append(_cond(pb.FilterCond(key=key, eq=_scalar(key, val))))
+        elif op == "$ne":
+            eq = _cond(pb.FilterCond(key=key, eq=_scalar(key, val)))
+            exprs.append(pb.FilterExpr(**{"not": eq}))
+        elif op == "$in":
+            if not isinstance(val, list):
+                raise errors.InvalidArgumentError(f"filter: key {key!r}: $in takes an array")
+            values = [_scalar(key, item) for item in val]
+            exprs.append(_cond(pb.FilterCond(key=key, **{"in": pb.ValueList(values=values)})))
+        elif op in _RANGE_OPS:
+            if isinstance(val, str):
+                lex_set = True
+            elif isinstance(val, (int, float)) and not isinstance(val, bool):
+                numeric_set = True
+            else:
+                raise errors.InvalidArgumentError(
+                    f"filter: key {key!r}: {op} takes a string or number"
+                )
+            bounds[_RANGE_OPS[op]] = val if isinstance(val, str) else str(val)
+        elif op == "$exists":
+            if not isinstance(val, bool):
+                raise errors.InvalidArgumentError(
+                    f"filter: key {key!r}: $exists takes a boolean"
+                )
+            exprs.append(_cond(pb.FilterCond(key=key, exists=val)))
+        else:
+            raise errors.InvalidArgumentError(
+                f"filter: key {key!r}: unknown operator {op!r}"
+            )
+    if bounds:
+        if numeric_set and lex_set:
+            raise errors.InvalidArgumentError(
+                f"filter: key {key!r}: range bounds mix numbers and strings"
+            )
+        rng = pb.FilterRange(numeric=numeric_set, **bounds)
+        exprs.append(_cond(pb.FilterCond(key=key, range=rng)))
+    collapsed = _and_collapse(exprs)
+    assert collapsed is not None  # ops was non-empty
+    return collapsed
+
+
+def _and_collapse(exprs: List["pb.FilterExpr"]) -> Optional["pb.FilterExpr"]:
+    if not exprs:
+        return None
+    if len(exprs) == 1:
+        return exprs[0]
+    return pb.FilterExpr(**{"and": pb.FilterJunction(exprs=exprs)})
+
+
+def _cond(c: "pb.FilterCond") -> "pb.FilterExpr":
+    return pb.FilterExpr(cond=c)
+
+
+def _scalar(key: str, val: Any) -> str:
+    """A scalar's metadata string form: strings as-is, numbers by their
+    literal ("0.5"), booleans as "true"/"false" — matching what the HTTP
+    client's JSON encoding would send."""
+    if isinstance(val, bool):  # before int: bool subclasses int
+        return "true" if val else "false"
+    if isinstance(val, str):
+        return val
+    if isinstance(val, (int, float)):
+        return str(val)
+    raise errors.InvalidArgumentError(
+        f"filter: key {key!r}: expected a string, number or boolean"
+    )