cairn-db-client 0.3.2__tar.gz

cairn_db_client-0.3.2/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+.venv/
+dist/
+.pytest_cache/

cairn_db_client-0.3.2/PKG-INFO

@@ -0,0 +1,102 @@
+Metadata-Version: 2.5
+Name: cairn-db-client
+Version: 0.3.2
+Summary: Client for Cairn, the hybrid search database where a deletion is final
+Project-URL: Homepage, https://github.com/Cairn-DB/cairn
+Project-URL: Repository, https://github.com/Cairn-DB/cairn
+Project-URL: Issues, https://github.com/Cairn-DB/cairn/issues
+Project-URL: Changelog, https://github.com/Cairn-DB/cairn/blob/main/CHANGELOG.md
+License-Expression: Apache-2.0
+Keywords: cairn,database,hybrid-search,rag,vector-search
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: httpx>=0.24
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == 'test'
+Description-Content-Type: text/markdown
+
+# cairn-db-client
+
+Python client for [Cairn](https://github.com/Cairn-DB/cairn), the hybrid search database
+where a deletion is final. It comes in two flavours, sync (`Client`) and async
+(`AsyncClient`), and is typed. Its only dependency is `httpx`.
+
+```bash
+pip install cairn-db-client
+```
+
+```python
+from cairn_db import Client, eq, range_, and_
+
+db = Client("http://localhost:7200", api_key=KEY)
+
+# Your own ids; chunks carry their parent in an ordinary field ("parent" by default).
+db.upsert([
+    {"id": "report-9#0", "parent": "report-9", "text": "...", "embedding": [...], "lang": "en"},
+    {"id": "report-9#1", "parent": "report-9", "text": "...", "embedding": [...], "lang": "en"},
+])
+
+hits = db.search(
+    k=10,
+    vector={"field": "embedding", "values": query_vector},
+    text="nuclear energy", text_field="text",
+    filter=and_(eq("lang", "en"), range_("year", gte=2020)),
+)
+for h in hits:
+    print(h.id, h.score, h.document["text"])
+
+db.delete(parent="report-9")               # the document and all its chunks
+db.delete(filter=eq("source", "crawler"))  # everything that matches, now
+db.delete(ids=["report-7#0"])
+```
+
+`AsyncClient` has the same methods, as coroutines (`async with AsyncClient(...) as db:`).
+
+## Read-your-writes and takedowns
+
+The client keeps the consistency token of its writes and takedowns, and sends it with every
+read. It therefore reads what it wrote, and never reads back what it deleted, through any
+node. To give the same guarantee to another service, pass it `db.token`:
+`Client(..., token=token)`, or `db.observe(token)` on an existing client.
+
+## Collections
+
+Without `collection()`, calls act on `default`, the collection defined at startup.
+
+```python
+admin = Client(url, api_key=ADMIN_KEY)
+admin.create_collection("notes", {"fields": [
+    {"name": "embedding", "kind": {"Vector": {"dims": 384, "metric": "Cosine"}}},
+    {"name": "body", "kind": "Text"},
+]}, shards=4)
+notes = db.collection("notes")   # the same calls, on "notes"
+notes.upsert([{"id": "n1", "embedding": [...], "body": "..."}])
+admin.drop_collection("notes")   # deletes its data on every node
+```
+
+`create_collection` and `drop_collection` need an admin key. `list_collections` needs a read key.
+
+## Tenants
+
+```python
+acme = db.with_tenant("acme")   # an unscoped key acting for tenant "acme"
+acme.upsert([{"id": "note-1", "text": "...", "embedding": [...]}])
+db.forget_tenant("acme")        # erase everything of "acme"
+```
+
+A key scoped to a tenant (`cairn-server keygen app read,write --tenant acme`) needs nothing
+else: every call stays inside its tenant. Ids belong to their tenant.
+
+## Errors and retries
+
+- `InvalidInputError` (400), `AuthenticationError` (401), `ForbiddenError` (403),
+  `UnavailableError` (503 or network), all subclasses of `CairnError` with a `status`.
+- `get` returns `None` for a missing document.
+- 502, 503, 504 and network errors are retried (`retries=3` by default) with backoff, on the
+  next address when `url` is a list.
+
+## Tests
+
+`PYTHONPATH=src python -m pytest tests` runs the unit tests. `../test-live.sh` also runs the
+live tests against a fresh local node.

cairn_db_client-0.3.2/README.md

@@ -0,0 +1,84 @@
+# cairn-db-client
+
+Python client for [Cairn](https://github.com/Cairn-DB/cairn), the hybrid search database
+where a deletion is final. It comes in two flavours, sync (`Client`) and async
+(`AsyncClient`), and is typed. Its only dependency is `httpx`.
+
+```bash
+pip install cairn-db-client
+```
+
+```python
+from cairn_db import Client, eq, range_, and_
+
+db = Client("http://localhost:7200", api_key=KEY)
+
+# Your own ids; chunks carry their parent in an ordinary field ("parent" by default).
+db.upsert([
+    {"id": "report-9#0", "parent": "report-9", "text": "...", "embedding": [...], "lang": "en"},
+    {"id": "report-9#1", "parent": "report-9", "text": "...", "embedding": [...], "lang": "en"},
+])
+
+hits = db.search(
+    k=10,
+    vector={"field": "embedding", "values": query_vector},
+    text="nuclear energy", text_field="text",
+    filter=and_(eq("lang", "en"), range_("year", gte=2020)),
+)
+for h in hits:
+    print(h.id, h.score, h.document["text"])
+
+db.delete(parent="report-9")               # the document and all its chunks
+db.delete(filter=eq("source", "crawler"))  # everything that matches, now
+db.delete(ids=["report-7#0"])
+```
+
+`AsyncClient` has the same methods, as coroutines (`async with AsyncClient(...) as db:`).
+
+## Read-your-writes and takedowns
+
+The client keeps the consistency token of its writes and takedowns, and sends it with every
+read. It therefore reads what it wrote, and never reads back what it deleted, through any
+node. To give the same guarantee to another service, pass it `db.token`:
+`Client(..., token=token)`, or `db.observe(token)` on an existing client.
+
+## Collections
+
+Without `collection()`, calls act on `default`, the collection defined at startup.
+
+```python
+admin = Client(url, api_key=ADMIN_KEY)
+admin.create_collection("notes", {"fields": [
+    {"name": "embedding", "kind": {"Vector": {"dims": 384, "metric": "Cosine"}}},
+    {"name": "body", "kind": "Text"},
+]}, shards=4)
+notes = db.collection("notes")   # the same calls, on "notes"
+notes.upsert([{"id": "n1", "embedding": [...], "body": "..."}])
+admin.drop_collection("notes")   # deletes its data on every node
+```
+
+`create_collection` and `drop_collection` need an admin key. `list_collections` needs a read key.
+
+## Tenants
+
+```python
+acme = db.with_tenant("acme")   # an unscoped key acting for tenant "acme"
+acme.upsert([{"id": "note-1", "text": "...", "embedding": [...]}])
+db.forget_tenant("acme")        # erase everything of "acme"
+```
+
+A key scoped to a tenant (`cairn-server keygen app read,write --tenant acme`) needs nothing
+else: every call stays inside its tenant. Ids belong to their tenant.
+
+## Errors and retries
+
+- `InvalidInputError` (400), `AuthenticationError` (401), `ForbiddenError` (403),
+  `UnavailableError` (503 or network), all subclasses of `CairnError` with a `status`.
+- `get` returns `None` for a missing document.
+- 502, 503, 504 and network errors are retried (`retries=3` by default) with backoff, on the
+  next address when `url` is a list.
+
+## Tests
+
+`PYTHONPATH=src python -m pytest tests` runs the unit tests. `../test-live.sh` also runs the
+live tests against a fresh local node.

cairn_db_client-0.3.2/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["hatchling>=1.24"]
+build-backend = "hatchling.build"
+
+[project]
+name = "cairn-db-client"
+version = "0.3.2"
+description = "Client for Cairn, the hybrid search database where a deletion is final"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.9"
+dependencies = ["httpx>=0.24"]
+classifiers = ["Typing :: Typed", "Programming Language :: Python :: 3"]
+
+keywords = ["cairn", "vector-search", "hybrid-search", "rag", "database"]
+
+[project.urls]
+Homepage = "https://github.com/Cairn-DB/cairn"
+Repository = "https://github.com/Cairn-DB/cairn"
+Issues = "https://github.com/Cairn-DB/cairn/issues"
+Changelog = "https://github.com/Cairn-DB/cairn/blob/main/CHANGELOG.md"
+
+[project.optional-dependencies]
+test = ["pytest>=7"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/cairn_db"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

cairn_db_client-0.3.2/src/cairn_db/__init__.py

@@ -0,0 +1,37 @@
+"""``cairn-db-client`` (imported as ``cairn_db``): the Cairn HTTP API from Python, sync and async (ADR 0031)."""
+
+from ._core import (
+    AuthenticationError,
+    CairnError,
+    Consistency,
+    DeleteResult,
+    Document,
+    Filter,
+    ForbiddenError,
+    Hit,
+    Id,
+    InvalidInputError,
+    LegHit,
+    NotFoundError,
+    UnavailableError,
+    VectorLeg,
+    WriteResult,
+    and_,
+    eq,
+    in_,
+    is_null,
+    merge_tokens,
+    not_,
+    or_,
+    range_,
+)
+from .client import AsyncClient, Client
+
+__version__ = "0.3.2"
+
+__all__ = [
+    "AsyncClient", "AuthenticationError", "CairnError", "Client", "Consistency", "DeleteResult",
+    "Document", "Filter", "ForbiddenError", "Hit", "Id", "InvalidInputError", "LegHit",
+    "NotFoundError", "UnavailableError", "VectorLeg", "WriteResult", "and_", "eq", "in_",
+    "is_null", "merge_tokens", "not_", "or_", "range_",
+]

cairn_db_client-0.3.2/src/cairn_db/_core.py

@@ -0,0 +1,283 @@
+"""Request building, tokens, results and errors shared by the sync and async clients."""
+
+from __future__ import annotations
+
+import re
+import urllib.parse
+from dataclasses import dataclass, field
+from typing import Any, Literal, Mapping, Optional, Sequence, TypedDict, Union
+
+Id = Union[str, int]
+"""A document id: your own string, or an unsigned integer below 2^63."""
+
+Document = dict[str, Any]
+"""A document: an ``id`` plus fields named as in the schema."""
+
+Filter = dict[str, Any]
+"""A filter: ``and``, ``or``, ``not``, or a condition on one field (see the helpers)."""
+
+Consistency = Literal["linearizable", "read_your_writes", "stale"]
+
+
+class VectorLeg(TypedDict, total=False):
+    field: str
+    values: list[float]
+    ef: int
+
+
+# ---------------------------------------------------------------- filters
+
+
+def eq(field: str, value: Any) -> Filter:
+    """``field`` equals ``value`` (on a Set field: contains it)."""
+    return {"field": field, "eq": value}
+
+
+def in_(field: str, values: Sequence[Any]) -> Filter:
+    """``field`` equals one of ``values`` (on a Set field: contains one of them)."""
+    return {"field": field, "in": list(values)}
+
+
+def range_(field: str, *, gt: Any = None, gte: Any = None, lt: Any = None, lte: Any = None) -> Filter:
+    """A numeric or date range."""
+    f: Filter = {"field": field}
+    for k, v in (("gt", gt), ("gte", gte), ("lt", lt), ("lte", lte)):
+        if v is not None:
+            f[k] = v
+    return f
+
+
+def is_null(field: str) -> Filter:
+    """``field`` has no value."""
+    return {"field": field, "is_null": True}
+
+
+def and_(*filters: Filter) -> Filter:
+    return {"and": list(filters)}
+
+
+def or_(*filters: Filter) -> Filter:
+    return {"or": list(filters)}
+
+
+def not_(f: Filter) -> Filter:
+    return {"not": f}
+
+
+# ---------------------------------------------------------------- errors
+
+
+class CairnError(Exception):
+    """Any error from the API or the network. ``status`` is 0 for a network error."""
+
+    def __init__(self, message: str, status: int) -> None:
+        super().__init__(message)
+        self.message = message
+        self.status = status
+
+
+class AuthenticationError(CairnError):
+    """401: missing or invalid API key."""
+
+
+class ForbiddenError(CairnError):
+    """403: the key lacks the role, or cannot act for that tenant."""
+
+
+class NotFoundError(CairnError):
+    """404."""
+
+
+class InvalidInputError(CairnError):
+    """400: invalid input (unknown field, wrong type, bad filter...)."""
+
+
+class UnavailableError(CairnError):
+    """503, or a network failure, after the retries."""
+
+
+def error_for(status: int, message: str) -> CairnError:
+    cls = {
+        400: InvalidInputError,
+        401: AuthenticationError,
+        403: ForbiddenError,
+        404: NotFoundError,
+        0: UnavailableError,
+        502: UnavailableError,
+        503: UnavailableError,
+        504: UnavailableError,
+    }.get(status, CairnError)
+    return cls(message, status)
+
+
+# ---------------------------------------------------------------- results
+
+
+@dataclass
+class WriteResult:
+    """Documents written (or ids given to a takedown by id), and the consistency token."""
+
+    count: int
+    token: str
+
+
+@dataclass
+class DeleteResult:
+    """``deleted``: documents removed (by filter, parent or tenant); ``count``: ids given."""
+
+    token: str
+    deleted: Optional[int] = None
+    count: Optional[int] = None
+
+
+@dataclass
+class LegHit:
+    rank: int
+    score: float
+
+
+@dataclass
+class Hit:
+    id: Id
+    score: float
+    legs: list[Optional[LegHit]] = field(default_factory=list)
+    document: Optional[Document] = None
+    tenant: Optional[str] = None
+    """The hit's tenant, shown to unscoped keys only."""
+    group: Any = None
+    """The hit's value of the ``group_by`` field."""
+
+    @staticmethod
+    def from_json(h: Mapping[str, Any]) -> "Hit":
+        return Hit(
+            id=h["id"],
+            score=h["score"],
+            legs=[LegHit(**leg) if leg else None for leg in h.get("legs", [])],
+            document=h.get("document"),
+            tenant=h.get("_tenant"),
+            group=h.get("group"),
+        )
+
+
+# ---------------------------------------------------------------- tokens and requests
+
+
+def merge_tokens(*tokens: Optional[str]) -> str:
+    """Per-shard maximum of consistency tokens (``shard.index,...``)."""
+    best: dict[int, int] = {}
+    for t in tokens:
+        for part in (t or "").split(","):
+            if not part:
+                continue
+            m = re.fullmatch(r"(\d+)\.(\d+)", part)
+            if not m:
+                raise InvalidInputError(f"bad consistency token {part!r}", 400)
+            s, i = int(m.group(1)), int(m.group(2))
+            best[s] = max(best.get(s, 0), i)
+    return ",".join(f"{s}.{i}" for s, i in sorted(best.items()))
+
+
+class TokenBox:
+    """Token state shared by a client and its tenant views."""
+
+    def __init__(self, value: str = "") -> None:
+        self.value = value
+
+    def observe(self, token: str) -> None:
+        self.value = merge_tokens(self.value, token)
+
+
+def collection_base(collection: Optional[str]) -> str:
+    """Path prefix of a collection's routes."""
+    if not collection or collection == "default":
+        return "/v1"
+    return "/v1/collections/" + urllib.parse.quote(collection, safe="")
+
+
+def doc_path(id: Id, after: str, consistency: Optional[str], base: str = "/v1") -> str:
+    """The path of a document: digits are an integer id, so a text id of digits says so."""
+    params: list[tuple[str, str]] = []
+    if isinstance(id, bool) or not isinstance(id, (str, int)):
+        raise InvalidInputError(f"an id is a string or an integer, not {id!r}", 400)
+    if isinstance(id, int):
+        if id < 0:
+            raise InvalidInputError(f"an integer id is non-negative: {id}", 400)
+        path = f"{base}/documents/{id}"
+    else:
+        path = f"{base}/documents/" + urllib.parse.quote(id, safe="")
+        if id.isdigit():
+            params.append(("id_type", "text"))
+    if after:
+        params.append(("after", after))
+    if consistency:
+        params.append(("consistency", consistency))
+    return path + ("?" + urllib.parse.urlencode(params) if params else "")
+
+
+def search_body(
+    *,
+    k: Optional[int],
+    vector: Union[VectorLeg, Sequence[VectorLeg], None],
+    text: Optional[str],
+    text_field: Optional[str],
+    all_terms: bool,
+    filter: Optional[Filter],
+    fusion: Optional[Mapping[str, Any]],
+    oversample: Optional[int],
+    with_documents: Optional[bool],
+    consistency: Optional[str],
+    after: str,
+    group_by: Optional[str] = None,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {}
+    if k is not None:
+        body["k"] = k
+    if isinstance(vector, Mapping):
+        body["vector"] = dict(vector)
+    elif vector:
+        body["vectors"] = [dict(v) for v in vector]
+    if text is not None:
+        if not text_field:
+            raise InvalidInputError("a text query needs text_field", 400)
+        body["text"] = {"field": text_field, "query": text, "all_terms": all_terms}
+    if filter is not None:
+        body["filter"] = filter
+    if fusion is not None:
+        body["fusion"] = dict(fusion)
+    if oversample is not None:
+        body["oversample"] = oversample
+    if with_documents is not None:
+        body["with_documents"] = with_documents
+    if consistency:
+        body["consistency"] = consistency
+    if group_by:
+        body["group_by"] = group_by
+    if after:
+        body["after"] = after
+    return body
+
+
+def delete_body(
+    *, ids: Optional[Sequence[Id]], filter: Optional[Filter], parent: Optional[Id], parent_field: str, after: str
+) -> dict[str, Any]:
+    if parent is not None:
+        if ids is not None or filter is not None:
+            raise InvalidInputError("parent cannot be combined with ids or filter", 400)
+        body: dict[str, Any] = {"filter": eq(parent_field, parent)}
+    elif ids is not None:
+        if not ids:
+            raise InvalidInputError("no ids", 400)
+        body = {"ids": list(ids)}
+        if filter is not None:
+            body["filter"] = filter
+    elif filter is not None:
+        body = {"filter": filter}
+    else:
+        raise InvalidInputError("give ids, filter or parent", 400)
+    if after:
+        body["after"] = after
+    return body
+
+
+def delete_result(r: Mapping[str, Any]) -> DeleteResult:
+    return DeleteResult(token=r["consistency_token"], deleted=r.get("deleted"), count=r.get("count"))

cairn_db_client-0.3.2/src/cairn_db/client.py

@@ -0,0 +1,403 @@
+"""Sync and async clients for the Cairn HTTP API (ADR 0031)."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+import urllib.parse
+from typing import Any, Mapping, Optional, Sequence, Union
+
+import httpx
+
+from ._core import (
+    Consistency,
+    DeleteResult,
+    Document,
+    Filter,
+    Hit,
+    Id,
+    NotFoundError,
+    TokenBox,
+    UnavailableError,
+    VectorLeg,
+    WriteResult,
+    collection_base,
+    delete_body,
+    delete_result,
+    doc_path,
+    error_for,
+    merge_tokens,
+    search_body,
+)
+
+
+class _Base:
+    def __init__(
+        self,
+        url: Union[str, Sequence[str]],
+        api_key: Optional[str] = None,
+        *,
+        tenant: Optional[str] = None,
+        parent_field: str = "parent",
+        timeout: float = 30.0,
+        retries: int = 3,
+        token: Optional[str] = None,
+        collection: Optional[str] = None,
+        _box: Optional[TokenBox] = None,
+    ) -> None:
+        urls = [url] if isinstance(url, str) else list(url)
+        if not urls:
+            raise ValueError("no url")
+        self._urls = [u.rstrip("/") for u in urls]
+        self._api_key = api_key
+        self._tenant = tenant
+        self._parent_field = parent_field
+        self._timeout = timeout
+        self._retries = retries
+        self._box = _box or TokenBox(merge_tokens(token) if token else "")
+        self._collection = collection
+        self._base = collection_base(collection)
+        self._next = 0
+
+    @property
+    def token(self) -> str:
+        """The consistency token covering every write and takedown of this client and its
+        tenant views. Hand it to another service so that its reads reflect them."""
+        return self._box.value
+
+    def observe(self, token: str) -> None:
+        """Adds a token received from elsewhere: later reads reflect those writes too."""
+        self._box.observe(token)
+
+    def _headers(self) -> dict[str, str]:
+        h = {"content-type": "application/json"}
+        if self._api_key:
+            h["authorization"] = f"Bearer {self._api_key}"
+        if self._tenant:
+            h["cairn-tenant"] = self._tenant
+        return h
+
+    def _view_args(self, **changes: Any) -> dict[str, Any]:
+        args: dict[str, Any] = dict(
+            url=self._urls,
+            api_key=self._api_key,
+            tenant=self._tenant,
+            parent_field=self._parent_field,
+            timeout=self._timeout,
+            retries=self._retries,
+            collection=self._collection,
+            _box=self._box,
+        )
+        args.update(changes)
+        return args
+
+    def _outcome(self, res: Optional[httpx.Response], err: Optional[Exception], url: str) -> Any:
+        """The decoded answer, or the error to raise (retry when it is an UnavailableError)."""
+        if res is None:
+            return UnavailableError(f"{url}: {err}", 0)
+        try:
+            data = res.json() if res.content else None
+        except ValueError:
+            data = None
+        if res.is_success:
+            return data
+        message = (data or {}).get("error") if isinstance(data, dict) else None
+        return error_for(res.status_code, message or res.text or res.reason_phrase)
+
+
+class Client(_Base):
+    """Blocking client.
+
+    >>> db = Client("http://localhost:7200", api_key=KEY)
+    >>> db.upsert([{"id": "doc-1#0", "parent": "doc-1", "text": "...", "embedding": [...]}])
+    >>> hits = db.search(text="nuclear", text_field="text", filter=eq("lang", "en"))
+    >>> db.delete(parent="doc-1")              # the document and all its chunks
+    >>> db.with_tenant("acme").get("doc-1")    # an unscoped key acting for one tenant
+    >>> db.collection("notes").search(...)      # another collection, same calls
+    >>> db.forget_tenant("acme")               # erase a tenant
+
+    Reads pass the token of this client's writes and takedowns, so it reads its own writes and
+    never reads back what it deleted, through any node.
+    """
+
+    def __init__(self, url: Union[str, Sequence[str]], api_key: Optional[str] = None, **kw: Any) -> None:
+        super().__init__(url, api_key, **kw)
+        self._http = httpx.Client(timeout=self._timeout)
+
+    def close(self) -> None:
+        self._http.close()
+
+    def __enter__(self) -> "Client":
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def with_tenant(self, tenant: str) -> "Client":
+        """A view acting for ``tenant`` (unscoped keys), sharing this client's token."""
+        return Client(**self._view_args(tenant=tenant))
+
+    def collection(self, name: str) -> "Client":
+        """A view acting on collection ``name``, sharing this client's token and tenant."""
+        return Client(**self._view_args(collection=name))
+
+    def create_collection(
+        self,
+        name: str,
+        schema: Mapping[str, Any],
+        *,
+        shards: Optional[int] = None,
+        expires_field: Optional[str] = None,
+    ) -> dict[str, Any]:
+        """Creates a collection (admin key); returns once it is ready on every node. With
+        ``expires_field`` (a field holding each document's expiry, Unix milliseconds), expired
+        documents are hidden at once and then deleted."""
+        body: dict[str, Any] = {"name": name, "schema": dict(schema)}
+        if shards is not None:
+            body["shards"] = shards
+        if expires_field:
+            body["expires_field"] = expires_field
+        return self._call("POST", "/v1/collections", body)
+
+    def list_collections(self) -> list[dict[str, Any]]:
+        """The live collections, ``default`` first."""
+        return self._call("GET", "/v1/collections")["collections"]
+
+    def drop_collection(self, name: str) -> None:
+        """Drops a collection and deletes its data on every node (admin key)."""
+        self._call("DELETE", "/v1/collections/" + urllib.parse.quote(name, safe=""))
+
+    def _call(self, method: str, path: str, body: Any = None) -> Any:
+        last: Exception = UnavailableError("no attempt", 0)
+        for attempt in range(self._retries + 1):
+            if attempt:
+                time.sleep(0.1 * 2 ** (attempt - 1))
+            url = self._urls[self._next % len(self._urls)] + path
+            res, err = None, None
+            try:
+                res = self._http.request(method, url, headers=self._headers(), json=body)
+            except httpx.HTTPError as e:
+                err = e
+            out = self._outcome(res, err, url)
+            if not isinstance(out, Exception):
+                return out
+            last = out
+            if not isinstance(out, UnavailableError):
+                raise out
+            self._next += 1
+        raise last
+
+    def upsert(self, documents: Sequence[Document]) -> WriteResult:
+        """Inserts or replaces documents."""
+        body: dict[str, Any] = {"documents": list(documents)}
+        if self.token:
+            body["after"] = self.token
+        r = self._call("POST", f"{self._base}/documents", body)
+        self.observe(r["consistency_token"])
+        return WriteResult(count=r["count"], token=r["consistency_token"])
+
+    def patch(self, id: Id, set: Mapping[str, Any]) -> WriteResult:
+        """Changes some fields of one document (``None`` clears a field). A missing document
+        is not created. ``count`` is the number of documents changed."""
+        return self.patch_many([{"id": id, "set": dict(set)}])
+
+    def patch_many(self, patches: Sequence[Mapping[str, Any]]) -> WriteResult:
+        """Changes fields of several documents: ``[{"id": ..., "set": {...}}]``."""
+        body: dict[str, Any] = {"patches": [dict(p) for p in patches]}
+        if self.token:
+            body["after"] = self.token
+        r = self._call("POST", f"{self._base}/documents/patch", body)
+        self.observe(r["consistency_token"])
+        return WriteResult(count=r["patched"], token=r["consistency_token"])
+
+    def get(self, id: Id, *, consistency: Optional[Consistency] = None) -> Optional[Document]:
+        """One document, or ``None`` when there is none (or it was taken down)."""
+        try:
+            return self._call("GET", doc_path(id, self.token, consistency, self._base))
+        except NotFoundError:
+            return None
+
+    def search(
+        self,
+        *,
+        k: Optional[int] = None,
+        vector: Union[VectorLeg, Sequence[VectorLeg], None] = None,
+        text: Optional[str] = None,
+        text_field: Optional[str] = None,
+        all_terms: bool = False,
+        filter: Optional[Filter] = None,
+        fusion: Optional[Mapping[str, Any]] = None,
+        oversample: Optional[int] = None,
+        with_documents: Optional[bool] = None,
+        consistency: Optional[Consistency] = None,
+        group_by: Optional[str] = None,
+    ) -> list[Hit]:
+        """Hybrid search: vector legs, a text leg and a filter, fused. ``group_by`` returns one
+        hit per value of that field (each document once, at its best chunk)."""
+        body = search_body(
+            k=k, vector=vector, text=text, text_field=text_field, all_terms=all_terms, filter=filter,
+            fusion=fusion, oversample=oversample, with_documents=with_documents,
+            consistency=consistency, after=self.token, group_by=group_by,
+        )
+        return [Hit.from_json(h) for h in self._call("POST", f"{self._base}/search", body)["hits"]]
+
+    def delete(
+        self,
+        *,
+        ids: Optional[Sequence[Id]] = None,
+        filter: Optional[Filter] = None,
+        parent: Optional[Id] = None,
+    ) -> DeleteResult:
+        """Takes documents down: by ``ids``, by ``filter`` (every document that matches when
+        the deletion is applied), by ``parent`` (a document's chunks), or ``ids`` that also
+        match ``filter``."""
+        body = delete_body(ids=ids, filter=filter, parent=parent, parent_field=self._parent_field, after=self.token)
+        r = self._call("POST", f"{self._base}/documents/delete", body)
+        self.observe(r["consistency_token"])
+        return delete_result(r)
+
+    def forget_tenant(self, tenant: str) -> DeleteResult:
+        """Erases a tenant: every document it holds (unscoped keys with the takedown role)."""
+        q = f"?after={urllib.parse.quote(self.token)}" if self.token else ""
+        r = self._call("DELETE", f"{self._base}/tenants/{urllib.parse.quote(tenant, safe='')}{q}")
+        self.observe(r["consistency_token"])
+        return delete_result(r)
+
+    def schema(self) -> dict[str, Any]:
+        """The collection schema (reserved fields hidden)."""
+        return self._call("GET", f"{self._base}/schema")
+
+
+class AsyncClient(_Base):
+    """The same API as :class:`Client`, with ``async`` methods."""
+
+    def __init__(self, url: Union[str, Sequence[str]], api_key: Optional[str] = None, **kw: Any) -> None:
+        super().__init__(url, api_key, **kw)
+        self._http = httpx.AsyncClient(timeout=self._timeout)
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+    async def __aenter__(self) -> "AsyncClient":
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.aclose()
+
+    def with_tenant(self, tenant: str) -> "AsyncClient":
+        """A view acting for ``tenant`` (unscoped keys), sharing this client's token."""
+        return AsyncClient(**self._view_args(tenant=tenant))
+
+    def collection(self, name: str) -> "AsyncClient":
+        """A view acting on collection ``name``, sharing this client's token and tenant."""
+        return AsyncClient(**self._view_args(collection=name))
+
+    async def create_collection(
+        self,
+        name: str,
+        schema: Mapping[str, Any],
+        *,
+        shards: Optional[int] = None,
+        expires_field: Optional[str] = None,
+    ) -> dict[str, Any]:
+        body: dict[str, Any] = {"name": name, "schema": dict(schema)}
+        if shards is not None:
+            body["shards"] = shards
+        if expires_field:
+            body["expires_field"] = expires_field
+        return await self._call("POST", "/v1/collections", body)
+
+    async def list_collections(self) -> list[dict[str, Any]]:
+        return (await self._call("GET", "/v1/collections"))["collections"]
+
+    async def drop_collection(self, name: str) -> None:
+        await self._call("DELETE", "/v1/collections/" + urllib.parse.quote(name, safe=""))
+
+    async def _call(self, method: str, path: str, body: Any = None) -> Any:
+        last: Exception = UnavailableError("no attempt", 0)
+        for attempt in range(self._retries + 1):
+            if attempt:
+                await asyncio.sleep(0.1 * 2 ** (attempt - 1))
+            url = self._urls[self._next % len(self._urls)] + path
+            res, err = None, None
+            try:
+                res = await self._http.request(method, url, headers=self._headers(), json=body)
+            except httpx.HTTPError as e:
+                err = e
+            out = self._outcome(res, err, url)
+            if not isinstance(out, Exception):
+                return out
+            last = out
+            if not isinstance(out, UnavailableError):
+                raise out
+            self._next += 1
+        raise last
+
+    async def upsert(self, documents: Sequence[Document]) -> WriteResult:
+        body: dict[str, Any] = {"documents": list(documents)}
+        if self.token:
+            body["after"] = self.token
+        r = await self._call("POST", f"{self._base}/documents", body)
+        self.observe(r["consistency_token"])
+        return WriteResult(count=r["count"], token=r["consistency_token"])
+
+    async def patch(self, id: Id, set: Mapping[str, Any]) -> WriteResult:
+        return await self.patch_many([{"id": id, "set": dict(set)}])
+
+    async def patch_many(self, patches: Sequence[Mapping[str, Any]]) -> WriteResult:
+        body: dict[str, Any] = {"patches": [dict(p) for p in patches]}
+        if self.token:
+            body["after"] = self.token
+        r = await self._call("POST", f"{self._base}/documents/patch", body)
+        self.observe(r["consistency_token"])
+        return WriteResult(count=r["patched"], token=r["consistency_token"])
+
+    async def get(self, id: Id, *, consistency: Optional[Consistency] = None) -> Optional[Document]:
+        try:
+            return await self._call("GET", doc_path(id, self.token, consistency, self._base))
+        except NotFoundError:
+            return None
+
+    async def search(
+        self,
+        *,
+        k: Optional[int] = None,
+        vector: Union[VectorLeg, Sequence[VectorLeg], None] = None,
+        text: Optional[str] = None,
+        text_field: Optional[str] = None,
+        all_terms: bool = False,
+        filter: Optional[Filter] = None,
+        fusion: Optional[Mapping[str, Any]] = None,
+        oversample: Optional[int] = None,
+        with_documents: Optional[bool] = None,
+        consistency: Optional[Consistency] = None,
+        group_by: Optional[str] = None,
+    ) -> list[Hit]:
+        body = search_body(
+            k=k, vector=vector, text=text, text_field=text_field, all_terms=all_terms, filter=filter,
+            fusion=fusion, oversample=oversample, with_documents=with_documents,
+            consistency=consistency, after=self.token, group_by=group_by,
+        )
+        r = await self._call("POST", f"{self._base}/search", body)
+        return [Hit.from_json(h) for h in r["hits"]]
+
+    async def delete(
+        self,
+        *,
+        ids: Optional[Sequence[Id]] = None,
+        filter: Optional[Filter] = None,
+        parent: Optional[Id] = None,
+    ) -> DeleteResult:
+        body = delete_body(ids=ids, filter=filter, parent=parent, parent_field=self._parent_field, after=self.token)
+        r = await self._call("POST", f"{self._base}/documents/delete", body)
+        self.observe(r["consistency_token"])
+        return delete_result(r)
+
+    async def forget_tenant(self, tenant: str) -> DeleteResult:
+        q = f"?after={urllib.parse.quote(self.token)}" if self.token else ""
+        r = await self._call("DELETE", f"{self._base}/tenants/{urllib.parse.quote(tenant, safe='')}{q}")
+        self.observe(r["consistency_token"])
+        return delete_result(r)
+
+    async def schema(self) -> dict[str, Any]:
+        return await self._call("GET", f"{self._base}/schema")

cairn_db_client-0.3.2/tests/test_live.py

@@ -0,0 +1,130 @@
+"""Against a running node (clients/test-live.sh): CAIRN_URL, CAIRN_ADMIN_KEY, CAIRN_KEY (read,write,takedown),
+CAIRN_ACME_KEY (scoped to tenant "acme"), CAIRN_READ_KEY (read only). Skipped without them."""
+
+import asyncio
+import os
+import time
+
+import pytest
+
+import cairn_db as c
+
+URL = os.environ.get("CAIRN_URL")
+pytestmark = pytest.mark.skipif(not URL, reason="CAIRN_URL not set")
+RUN = f"py-{os.getpid()}-{int(time.time() * 1000)}"
+
+
+def vec(i):
+    return [1.0, float(i), 0.5, 0.0]
+
+
+def test_client_against_a_live_node():
+    root = c.Client(URL.split(","), os.environ["CAIRN_KEY"])
+    db = root.with_tenant(RUN)  # a fresh tenant: this run's data only
+    chunks = [{"id": f"doc-1#{i}", "parent": "doc-1", "text": f"ocelot chunk {i}", "embedding": vec(i), "n": i} for i in range(4)]
+    chunks += [
+        {"id": "doc-2#0", "parent": "doc-2", "text": "ocelot other", "embedding": vec(9), "n": 9, "tags": ["x"]},
+        {"id": 42, "text": "integer id", "embedding": vec(4), "n": 42},
+        {"id": "007", "text": "digits text id", "embedding": vec(5), "n": 7},
+    ]
+    assert db.upsert(chunks).count == 7
+
+    assert db.get("doc-1#2") == {"id": "doc-1#2", "parent": "doc-1", "text": "ocelot chunk 2", "embedding": vec(2), "n": 2}
+    assert db.get(42)["text"] == "integer id"
+    assert db.get("007")["text"] == "digits text id"
+    assert db.get("missing") is None
+    hits = db.search(k=20, text="ocelot", text_field="text")
+    assert sorted(h.id for h in hits) == ["doc-1#0", "doc-1#1", "doc-1#2", "doc-1#3", "doc-2#0"]
+    assert all(h.document and h.tenant is None for h in hits)
+    hits = db.search(k=3, vector={"field": "embedding", "values": vec(4)}, filter=c.range_("n", gte=4), with_documents=False)
+    assert hits[0].id == 42
+
+    other = c.Client(URL, os.environ["CAIRN_KEY"], tenant=RUN, token=db.token)
+    assert other.get("doc-2#0")["parent"] == "doc-2"
+
+    assert db.patch("doc-1#2", {"text": "ocelot patched", "n": None}).count == 1
+    assert db.get("doc-1#2") == {"id": "doc-1#2", "parent": "doc-1", "text": "ocelot patched", "embedding": vec(2)}
+    assert db.patch_many([{"id": 42, "set": {"n": 43}}, {"id": "nope", "set": {"n": 1}}]).count == 1
+    assert db.get(42)["n"] == 43
+    with pytest.raises(c.InvalidInputError):
+        db.patch(42, {"nope": 1})
+    assert db.delete(parent="doc-1").deleted == 4
+    assert db.get("doc-1#0") is None
+    assert [h.id for h in db.search(k=20, text="ocelot", text_field="text")] == ["doc-2#0"]
+    assert db.delete(ids=[42, "007"], filter=c.eq("n", 7)).deleted == 1
+    assert db.get("007") is None and db.get(42)["n"] == 43
+    assert db.delete(ids=[42]).count == 1
+    assert db.get(42) is None
+
+    with pytest.raises(c.InvalidInputError):
+        db.upsert([{"id": 1, "nope": 1}])
+    with pytest.raises(c.AuthenticationError):
+        c.Client(URL, "cairn_wrong_key").schema()
+    with pytest.raises(c.ForbiddenError):
+        c.Client(URL, os.environ["CAIRN_READ_KEY"]).upsert([{"id": 1}])
+
+    acme = c.Client(URL, os.environ["CAIRN_ACME_KEY"])
+    acme.upsert([{"id": "shared", "text": "acme's", "embedding": vec(1)}])
+    db.upsert([{"id": "shared", "text": "run's", "embedding": vec(1)}])
+    assert acme.get("shared")["text"] == "acme's"
+    assert db.get("shared")["text"] == "run's"
+    with pytest.raises(c.ForbiddenError):
+        acme.with_tenant("globex").get("shared")
+    with pytest.raises(c.ForbiddenError):
+        acme.forget_tenant("acme")
+
+    assert root.forget_tenant(RUN).deleted == 2
+    assert db.search(k=10) == []
+    assert root.forget_tenant("acme").deleted >= 1
+    assert acme.get("shared") is None
+
+
+def test_async_client_against_a_live_node():
+    async def run():
+        async with c.AsyncClient(URL, os.environ["CAIRN_KEY"], tenant=RUN + "-async") as db:
+            await db.upsert([{"id": f"a{i}", "parent": "p", "text": "lynx", "embedding": vec(i)} for i in range(3)])
+            assert (await db.get("a1"))["text"] == "lynx"
+            assert len(await db.search(text="lynx", text_field="text")) == 3
+            assert (await db.delete(parent="p")).deleted == 3
+            assert await db.search(text="lynx", text_field="text") == []
+
+    asyncio.run(run())
+
+
+def test_collections_against_a_live_node():
+    admin = c.Client(URL, os.environ["CAIRN_ADMIN_KEY"])
+    name = f"py-notes-{os.getpid()}"
+    schema = {"fields": [{"name": "embedding", "kind": {"Vector": {"dims": 2, "metric": "L2"}}},
+                         {"name": "body", "kind": "Text"}, {"name": "parent", "kind": "Enum"}]}
+    assert admin.create_collection(name, schema, shards=2) == {"name": name, "shards": 2, "schema": schema}
+    assert name in [x["name"] for x in admin.list_collections()]
+    app = c.Client(URL, os.environ["CAIRN_KEY"])
+    notes = app.collection(name)
+    notes.upsert([{"id": f"n{i}", "embedding": [float(i), 1.0], "body": f"walrus {i}", "parent": f"p{i % 2}"} for i in range(4)])
+    assert notes.get("n1")["body"] == "walrus 1"
+    assert app.get("n1") is None, "not in default"
+    assert len(notes.search(text="walrus", text_field="body")) == 4
+    grouped = notes.search(vector={"field": "embedding", "values": [3.0, 1.0]}, group_by="parent")
+    assert [(h.id, h.group) for h in grouped] == [("n3", "p1"), ("n2", "p0")]
+    assert notes.delete(parent="p0").deleted == 2
+    assert sorted(h.id for h in notes.search(k=10)) == ["n1", "n3"]
+    assert notes.schema() == schema
+    with pytest.raises(c.ForbiddenError):
+        app.create_collection("x", schema)
+    admin.drop_collection(name)
+    assert name not in [x["name"] for x in admin.list_collections()]
+    with pytest.raises(c.NotFoundError):
+        notes.search(k=1)
+
+
+def test_retention_through_the_client():
+    admin = c.Client(URL, os.environ["CAIRN_ADMIN_KEY"])
+    name = f"py-ttl-{os.getpid()}"
+    schema = {"fields": [{"name": "body", "kind": "Text"}, {"name": "until", "kind": "I64"}]}
+    assert admin.create_collection(name, schema, expires_field="until")["expires_field"] == "until"
+    col = c.Client(URL, os.environ["CAIRN_KEY"]).collection(name)
+    now = int(time.time() * 1000)
+    col.upsert([{"id": "gone", "body": "x", "until": now - 1000}, {"id": "kept", "body": "x", "until": now + 600_000}])
+    assert col.get("gone") is None
+    assert [h.id for h in col.search(k=10)] == ["kept"]
+    admin.drop_collection(name)

cairn_db_client-0.3.2/tests/test_unit.py

@@ -0,0 +1,109 @@
+"""Request shapes, token tracking, retries and typed errors, on a mock transport."""
+
+import asyncio
+import json
+
+import httpx
+import pytest
+
+import cairn_db as c
+
+
+def mock(db, responses, calls):
+    def handler(req: httpx.Request) -> httpx.Response:
+        calls.append(req)
+        r = responses.pop(0)
+        if isinstance(r, Exception):
+            raise r
+        status, body = r
+        return httpx.Response(status, json=body)
+
+    if isinstance(db, c.AsyncClient):
+        db._http = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    else:
+        db._http = httpx.Client(transport=httpx.MockTransport(handler))
+    return db
+
+
+def test_tokens_and_filters():
+    assert c.merge_tokens("0.5,1.2", "1.7,2.1", None, "") == "0.5,1.7,2.1"
+    with pytest.raises(c.InvalidInputError):
+        c.merge_tokens("x")
+    assert c.and_(c.eq("s", "tv"), c.or_(c.in_("t", ["a"]), c.not_(c.is_null("d"))), c.range_("n", gte=1, lt=5)) == {
+        "and": [
+            {"field": "s", "eq": "tv"},
+            {"or": [{"field": "t", "in": ["a"]}, {"not": {"field": "d", "is_null": True}}]},
+            {"field": "n", "gte": 1, "lt": 5},
+        ]
+    }
+
+
+def test_requests_and_token_tracking():
+    calls = []
+    db = mock(
+        c.Client("http://n1/", "k", parent_field="doc"),
+        [
+            (200, {"count": 2, "consistency_token": "0.4,1.9"}),
+            (200, {"id": "12"}),
+            (200, {"hits": [{"id": "a", "score": 0.5, "legs": [{"rank": 1, "score": 2.0}, None], "_tenant": "acme"}]}),
+            (200, {"deleted": 3, "consistency_token": "1.12"}),
+            (404, {"error": "no document"}),
+            (200, {"deleted": 5, "consistency_token": "0.20"}),
+        ],
+        calls,
+    )
+    assert db.upsert([{"id": "a"}, {"id": 3}]) == c.WriteResult(count=2, token="0.4,1.9")
+    assert calls[0].headers["authorization"] == "Bearer k"
+    db.get("12")
+    assert str(calls[1].url) == "http://n1/v1/documents/12?id_type=text&after=0.4%2C1.9"
+    hits = db.search(k=5, text="q", text_field="text", all_terms=True, vector=[{"field": "e", "values": [1.0]}],
+                     with_documents=False, filter=c.eq("a", 1))
+    assert json.loads(calls[2].content) == {
+        "k": 5, "vectors": [{"field": "e", "values": [1.0]}], "text": {"field": "text", "query": "q", "all_terms": True},
+        "filter": {"field": "a", "eq": 1}, "with_documents": False, "after": "0.4,1.9"}
+    assert hits == [c.Hit(id="a", score=0.5, legs=[c.LegHit(1, 2.0), None], tenant="acme")]
+    assert db.delete(parent="doc-9") == c.DeleteResult(token="1.12", deleted=3)
+    assert json.loads(calls[3].content) == {"filter": {"field": "doc", "eq": "doc-9"}, "after": "0.4,1.9"}
+    assert db.token == "0.4,1.12"
+    assert db.get("a/b c") is None
+    assert str(calls[4].url).startswith("http://n1/v1/documents/a%2Fb%20c?after=")
+    acme = db.with_tenant("acme")
+    acme._http = db._http
+    acme.forget_tenant("globex")
+    assert calls[5].headers["cairn-tenant"] == "acme"
+    assert str(calls[5].url) == "http://n1/v1/tenants/globex?after=0.4%2C1.12"
+    assert db.token == "0.20,1.12", "tenant views share the token"
+    with pytest.raises(c.InvalidInputError):
+        db.delete()
+    with pytest.raises(c.InvalidInputError):
+        db.delete(parent="x", ids=[1])
+
+
+def test_retries_and_errors():
+    calls = []
+    db = mock(c.Client(["http://a", "http://b"], retries=3), [
+        (503, {"error": "no leader"}), httpx.ConnectError("refused"), (200, {"count": 1, "consistency_token": "0.1"})
+    ], calls)
+    db.upsert([{"id": 1}])
+    assert [r.url.host for r in calls] == ["a", "b", "a"]
+    for status, cls in [(400, c.InvalidInputError), (401, c.AuthenticationError), (403, c.ForbiddenError), (500, c.CairnError)]:
+        db = mock(c.Client("http://a"), [(status, {"error": "boom"})], [])
+        with pytest.raises(cls) as e:
+            db.upsert([{"id": 1}])
+        assert (e.value.status, e.value.message) == (status, "boom")
+    db = mock(c.Client("http://a", retries=1), [(503, {}), (503, {})], [])
+    with pytest.raises(c.UnavailableError):
+        db.search()
+
+
+def test_async_client():
+    async def run():
+        calls = []
+        db = mock(c.AsyncClient("http://a"), [
+            (503, {}), (200, {"count": 1, "consistency_token": "2.3"}), (200, {"hits": []}),
+        ], calls)
+        assert (await db.upsert([{"id": "x"}])).token == "2.3"
+        assert await db.search(filter=c.eq("a", 1)) == []
+        assert json.loads(calls[2].content)["after"] == "2.3"
+        await db.aclose()
+    asyncio.run(run())