langchain-cairn 0.3.2__tar.gz

langchain_cairn-0.3.2/.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+*.pyc
+.pytest_cache/
+dist/

langchain_cairn-0.3.2/PKG-INFO

@@ -0,0 +1,54 @@
+Metadata-Version: 2.5
+Name: langchain-cairn
+Version: 0.3.2
+Summary: LangChain vector store for Cairn, the hybrid search database where a deletion is final
+Project-URL: Homepage, https://github.com/Cairn-DB/cairn
+Project-URL: Repository, https://github.com/Cairn-DB/cairn
+Project-URL: Issues, https://github.com/Cairn-DB/cairn/issues
+Project-URL: Changelog, https://github.com/Cairn-DB/cairn/blob/main/CHANGELOG.md
+License-Expression: Apache-2.0
+Keywords: cairn,database,hybrid-search,langchain,rag,vector-search
+Requires-Python: >=3.10
+Requires-Dist: cairn-db-client>=0.3.2
+Requires-Dist: langchain-core>=0.3
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == 'test'
+Description-Content-Type: text/markdown
+
+# langchain-cairn
+
+LangChain vector store for [Cairn](https://github.com/Cairn-DB/cairn), the hybrid search
+database where a deletion is final.
+
+```bash
+pip install langchain-cairn
+```
+
+```python
+from cairn_db import Client
+from langchain_cairn import CairnVectorStore, collection_schema
+
+admin = Client("http://localhost:7200", api_key=ADMIN_KEY)
+admin.create_collection("docs", collection_schema(1536, filterable={"source": "Enum"}))
+
+store = CairnVectorStore(Client("http://localhost:7200", api_key=KEY).collection("docs"), embeddings)
+store.add_documents(chunks)                      # chunk.metadata["parent"] = the source document
+store.similarity_search("what changed in 2024?", k=4, filter={"source": "handbook"})
+store.similarity_search("...", k=4, group_by="parent")   # each document once
+store.hybrid_search("error E1234", k=4)          # vector + BM25, fused
+store.as_retriever()
+store.delete(parent="handbook.pdf")              # the document and all its chunks, everywhere
+```
+
+- **Schema.** `collection_schema` creates the fields the store needs:
+  - the text and the embedding;
+  - `parent`;
+  - `metadata`, which holds each document's whole metadata as JSON, so any metadata
+    round-trips;
+  - the `filterable` fields, which store the same metadata keys so they can be filtered.
+- **Filters.** Either a plain dict (`{"source": "a"}`: every key equals its value), or a
+  Cairn filter (`{"field": "page", "gte": 2}`, `and`/`or`/`not`).
+- **Scores.** `similarity_search_with_score` returns the similarity for Cosine and Dot, and
+  the squared distance for L2. `similarity_search_with_relevance_scores` maps them to [0, 1].
+- **Deletions.** A deletion by ids, by parent or by filter is final. The client carries its
+  consistency token, so later searches never return the deleted documents, on any node.

langchain_cairn-0.3.2/README.md

@@ -0,0 +1,37 @@
+# langchain-cairn
+
+LangChain vector store for [Cairn](https://github.com/Cairn-DB/cairn), the hybrid search
+database where a deletion is final.
+
+```bash
+pip install langchain-cairn
+```
+
+```python
+from cairn_db import Client
+from langchain_cairn import CairnVectorStore, collection_schema
+
+admin = Client("http://localhost:7200", api_key=ADMIN_KEY)
+admin.create_collection("docs", collection_schema(1536, filterable={"source": "Enum"}))
+
+store = CairnVectorStore(Client("http://localhost:7200", api_key=KEY).collection("docs"), embeddings)
+store.add_documents(chunks)                      # chunk.metadata["parent"] = the source document
+store.similarity_search("what changed in 2024?", k=4, filter={"source": "handbook"})
+store.similarity_search("...", k=4, group_by="parent")   # each document once
+store.hybrid_search("error E1234", k=4)          # vector + BM25, fused
+store.as_retriever()
+store.delete(parent="handbook.pdf")              # the document and all its chunks, everywhere
+```
+
+- **Schema.** `collection_schema` creates the fields the store needs:
+  - the text and the embedding;
+  - `parent`;
+  - `metadata`, which holds each document's whole metadata as JSON, so any metadata
+    round-trips;
+  - the `filterable` fields, which store the same metadata keys so they can be filtered.
+- **Filters.** Either a plain dict (`{"source": "a"}`: every key equals its value), or a
+  Cairn filter (`{"field": "page", "gte": 2}`, `and`/`or`/`not`).
+- **Scores.** `similarity_search_with_score` returns the similarity for Cosine and Dot, and
+  the squared distance for L2. `similarity_search_with_relevance_scores` maps them to [0, 1].
+- **Deletions.** A deletion by ids, by parent or by filter is final. The client carries its
+  consistency token, so later searches never return the deleted documents, on any node.

langchain_cairn-0.3.2/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["hatchling>=1.24"]
+build-backend = "hatchling.build"
+
+[project]
+name = "langchain-cairn"
+version = "0.3.2"
+description = "LangChain vector store for Cairn, the hybrid search database where a deletion is final"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+dependencies = ["cairn-db-client>=0.3.2", "langchain-core>=0.3"]
+
+keywords = ["cairn", "vector-search", "hybrid-search", "rag", "database", "langchain"]
+
+[project.urls]
+Homepage = "https://github.com/Cairn-DB/cairn"
+Repository = "https://github.com/Cairn-DB/cairn"
+Issues = "https://github.com/Cairn-DB/cairn/issues"
+Changelog = "https://github.com/Cairn-DB/cairn/blob/main/CHANGELOG.md"
+
+[project.optional-dependencies]
+test = ["pytest>=7"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/langchain_cairn"]

langchain_cairn-0.3.2/src/langchain_cairn/__init__.py

@@ -0,0 +1,5 @@
+"""LangChain vector store for Cairn (ADR 0031)."""
+
+from .vectorstores import CairnVectorStore, collection_schema
+
+__all__ = ["CairnVectorStore", "collection_schema"]

langchain_cairn-0.3.2/src/langchain_cairn/vectorstores.py

@@ -0,0 +1,242 @@
+"""``CairnVectorStore``: LangChain's vector store interface over a Cairn collection.
+
+Documents are stored with the text in ``text_field``, the embedding in ``vector_field``, the
+whole metadata as JSON in the ``metadata`` field (so any metadata round-trips), and the
+metadata keys that are also fields of the schema in those fields (so they can be filtered).
+A document's ``parent`` metadata, if the schema has a ``parent`` field, lets
+``delete(parent=...)`` remove a document and all its chunks.
+
+Deletions are final: once ``delete`` returns, no search or read through this store returns
+the documents again, on any node (the client passes its consistency token).
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import math
+import uuid
+from typing import Any, Callable, Iterable, Optional, Sequence
+
+from cairn_db import Client, and_, eq
+from langchain_core.documents import Document
+from langchain_core.embeddings import Embeddings
+from langchain_core.vectorstores import VectorStore
+
+METADATA_FIELD = "metadata"
+
+
+def collection_schema(
+    dims: int,
+    *,
+    metric: str = "Cosine",
+    text_field: str = "text",
+    vector_field: str = "embedding",
+    filterable: Optional[dict[str, str]] = None,
+) -> dict[str, Any]:
+    """A schema for a store: text, embedding, ``parent``, the full metadata as JSON, and
+    ``filterable`` metadata fields (name -> kind: ``Enum``, ``I64``, ``F64``, ``Date``,
+    ``Bool``, ``Set`` or ``Text``)."""
+    fields = [
+        {"name": vector_field, "kind": {"Vector": {"dims": dims, "metric": metric}}},
+        {"name": text_field, "kind": "Text"},
+        {"name": "parent", "kind": "Enum"},
+        {"name": METADATA_FIELD, "kind": "Blob"},
+    ]
+    for name, kind in (filterable or {}).items():
+        fields.append({"name": name, "kind": kind})
+    return {"fields": fields}
+
+
+def _filter(f: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
+    """A Cairn filter, or ``{"field": value, ...}`` meaning every field equals its value."""
+    if not f:
+        return None
+    if any(k in f for k in ("and", "or", "not", "field")):
+        return f
+    parts = [eq(k, v) for k, v in f.items()]
+    return parts[0] if len(parts) == 1 else and_(*parts)
+
+
+class CairnVectorStore(VectorStore):
+    """A Cairn collection as a LangChain vector store.
+
+    >>> from cairn_db import Client
+    >>> client = Client("http://localhost:7200", api_key=KEY)
+    >>> client.create_collection("docs", collection_schema(1536, filterable={"source": "Enum"}))
+    >>> store = CairnVectorStore(client.collection("docs"), embeddings)
+    >>> store.add_documents(chunks)
+    >>> store.similarity_search("what changed in 2024?", k=4, filter={"source": "handbook"})
+    >>> store.delete(parent="handbook.pdf")   # the document and all its chunks
+    """
+
+    def __init__(
+        self,
+        client: Client,
+        embedding: Embeddings,
+        *,
+        text_field: str = "text",
+        vector_field: str = "embedding",
+    ) -> None:
+        self.client = client
+        self._embedding = embedding
+        self.text_field = text_field
+        self.vector_field = vector_field
+        fields = {f["name"]: f["kind"] for f in client.schema()["fields"]}
+        if vector_field not in fields or text_field not in fields:
+            raise ValueError(f"the collection needs a {vector_field!r} and a {text_field!r} field")
+        self.metric = fields[vector_field]["Vector"]["metric"]
+        self._fields = fields
+
+    @property
+    def embeddings(self) -> Embeddings:
+        return self._embedding
+
+    # ------------------------------------------------------------------ writes
+
+    def _doc(self, id: str, text: str, vector: list[float], metadata: dict[str, Any]) -> dict[str, Any]:
+        d: dict[str, Any] = {"id": id, self.text_field: text, self.vector_field: vector}
+        if METADATA_FIELD in self._fields:
+            d[METADATA_FIELD] = base64.b64encode(json.dumps(metadata, default=str).encode()).decode()
+        for k, v in metadata.items():
+            if k in self._fields and k not in (self.text_field, self.vector_field, METADATA_FIELD):
+                d[k] = v
+        return d
+
+    def add_texts(
+        self,
+        texts: Iterable[str],
+        metadatas: Optional[list[dict[str, Any]]] = None,
+        *,
+        ids: Optional[list[str]] = None,
+        batch_size: int = 256,
+        **kwargs: Any,
+    ) -> list[str]:
+        texts = list(texts)
+        metadatas = metadatas or [{} for _ in texts]
+        ids = ids or [str(uuid.uuid4()) for _ in texts]
+        vectors = self._embedding.embed_documents(texts)
+        for i in range(0, len(texts), batch_size):
+            self.client.upsert(
+                [self._doc(ids[j], texts[j], vectors[j], metadatas[j]) for j in range(i, min(i + batch_size, len(texts)))]
+            )
+        return ids
+
+    def add_documents(self, documents: list[Document], **kwargs: Any) -> list[str]:
+        ids = kwargs.pop("ids", None) or [d.id or str(uuid.uuid4()) for d in documents]
+        return self.add_texts([d.page_content for d in documents], [d.metadata for d in documents], ids=ids, **kwargs)
+
+    def delete(self, ids: Optional[list[str]] = None, **kwargs: Any) -> Optional[bool]:
+        """Deletes ``ids``, or with ``parent=`` a document and all its chunks, or with
+        ``filter=`` every document that matches."""
+        if ids:
+            self.client.delete(ids=list(ids))
+        elif "parent" in kwargs:
+            self.client.delete(parent=kwargs["parent"])
+        elif kwargs.get("filter"):
+            self.client.delete(filter=_filter(kwargs["filter"]))
+        else:
+            raise ValueError("give ids, parent= or filter=")
+        return True
+
+    # ------------------------------------------------------------------ reads
+
+    def _to_document(self, id: Any, doc: dict[str, Any]) -> Document:
+        metadata: dict[str, Any] = {}
+        raw = doc.get(METADATA_FIELD)
+        if raw:
+            try:
+                metadata = json.loads(base64.b64decode(raw))
+            except ValueError:
+                metadata = {}
+        else:
+            metadata = {k: v for k, v in doc.items() if k not in ("id", self.text_field, self.vector_field)}
+        return Document(id=str(id), page_content=doc.get(self.text_field, ""), metadata=metadata)
+
+    def get_by_ids(self, ids: Sequence[str], /) -> list[Document]:
+        out = []
+        for i in ids:
+            d = self.client.get(i)
+            if d is not None:
+                out.append(self._to_document(i, d))
+        return out
+
+    def _score(self, hit: Any) -> float:
+        """Similarity for Cosine and Dot (higher is better), squared distance for L2."""
+        leg = hit.legs[0] if hit.legs else None
+        if leg is None:
+            return float("nan")
+        return leg.score if self.metric == "L2" else -leg.score
+
+    def similarity_search_by_vector_with_score(
+        self,
+        embedding: list[float],
+        k: int = 4,
+        filter: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> list[tuple[Document, float]]:
+        hits = self.client.search(
+            k=k,
+            vector={"field": self.vector_field, "values": list(embedding)},
+            filter=_filter(filter),
+            group_by=kwargs.get("group_by"),
+        )
+        return [(self._to_document(h.id, h.document or {}), self._score(h)) for h in hits]
+
+    def similarity_search_with_score(
+        self, query: str, k: int = 4, filter: Optional[dict[str, Any]] = None, **kwargs: Any
+    ) -> list[tuple[Document, float]]:
+        """Nearest documents to ``query``. ``group_by="parent"`` returns each document once."""
+        return self.similarity_search_by_vector_with_score(
+            self._embedding.embed_query(query), k, filter, **kwargs
+        )
+
+    def similarity_search(
+        self, query: str, k: int = 4, filter: Optional[dict[str, Any]] = None, **kwargs: Any
+    ) -> list[Document]:
+        return [d for d, _ in self.similarity_search_with_score(query, k, filter, **kwargs)]
+
+    def similarity_search_by_vector(
+        self, embedding: list[float], k: int = 4, filter: Optional[dict[str, Any]] = None, **kwargs: Any
+    ) -> list[Document]:
+        return [d for d, _ in self.similarity_search_by_vector_with_score(embedding, k, filter, **kwargs)]
+
+    def hybrid_search(
+        self, query: str, k: int = 4, filter: Optional[dict[str, Any]] = None, **kwargs: Any
+    ) -> list[Document]:
+        """Vector and BM25 legs fused (RRF): exact words and meaning together."""
+        hits = self.client.search(
+            k=k,
+            vector={"field": self.vector_field, "values": self._embedding.embed_query(query)},
+            text=query,
+            text_field=self.text_field,
+            filter=_filter(filter),
+            group_by=kwargs.get("group_by"),
+        )
+        return [self._to_document(h.id, h.document or {}) for h in hits]
+
+    def _select_relevance_score_fn(self) -> Callable[[float], float]:
+        if self.metric == "Cosine":
+            return lambda s: (1.0 + s) / 2.0
+        if self.metric == "L2":
+            return lambda d: 1.0 - math.sqrt(max(d, 0.0)) / math.sqrt(2)
+        return self._max_inner_product_relevance_score_fn
+
+    @classmethod
+    def from_texts(
+        cls,
+        texts: list[str],
+        embedding: Embeddings,
+        metadatas: Optional[list[dict[str, Any]]] = None,
+        *,
+        ids: Optional[list[str]] = None,
+        client: Optional[Client] = None,
+        **kwargs: Any,
+    ) -> "CairnVectorStore":
+        """A store over ``client`` (a :class:`cairn_db.Client`, or a collection view of one)
+        holding ``texts``."""
+        if client is None:
+            raise ValueError("from_texts needs client=cairn_db.Client(...)")
+        store = cls(client, embedding, **kwargs)
+        store.add_texts(texts, metadatas, ids=ids)
+        return store

langchain_cairn-0.3.2/tests/test_langchain_live.py

@@ -0,0 +1,83 @@
+"""Against a running node (clients/test-live.sh): CAIRN_URL, CAIRN_ADMIN_KEY, CAIRN_KEY."""
+
+import hashlib
+import math
+import os
+
+import pytest
+from cairn_db import Client, NotFoundError
+from langchain_core.documents import Document
+from langchain_core.embeddings import Embeddings
+
+from langchain_cairn import CairnVectorStore, collection_schema
+
+URL = os.environ.get("CAIRN_URL")
+pytestmark = pytest.mark.skipif(not URL, reason="CAIRN_URL not set")
+DIMS = 64
+
+
+class WordHash(Embeddings):
+    """Words hashed into 64 dimensions: texts sharing words are close."""
+
+    def _one(self, text: str) -> list[float]:
+        v = [0.0] * DIMS
+        for w in text.lower().split():
+            h = hashlib.sha1(w.encode()).digest()
+            v[h[0] % DIMS] += 1.0 if h[1] & 1 else -1.0
+        n = math.sqrt(sum(x * x for x in v)) or 1.0
+        return [x / n for x in v]
+
+    def embed_documents(self, texts):
+        return [self._one(t) for t in texts]
+
+    def embed_query(self, text):
+        return self._one(text)
+
+
+def test_langchain_store_against_a_live_node():
+    admin = Client(URL, os.environ["CAIRN_ADMIN_KEY"])
+    name = f"lc-{os.getpid()}"
+    admin.create_collection(name, collection_schema(DIMS, filterable={"source": "Enum", "page": "I64"}), shards=2)
+    try:
+        store = CairnVectorStore(Client(URL, os.environ["CAIRN_KEY"]).collection(name), WordHash())
+        chunks = [
+            Document(page_content="the otter swims in the river", metadata={"source": "a", "parent": "doc-a", "page": 1, "extra": {"k": [1, 2]}}),
+            Document(page_content="an otter eats fish near the river bank", metadata={"source": "a", "parent": "doc-a", "page": 2}),
+            Document(page_content="the stock market fell sharply today", metadata={"source": "b", "parent": "doc-b", "page": 1}),
+            Document(page_content="markets and stocks after the election", metadata={"source": "b", "parent": "doc-b", "page": 2}),
+        ]
+        ids = store.add_documents(chunks, ids=["a1", "a2", "b1", "b2"])
+        assert ids == ["a1", "a2", "b1", "b2"]
+
+        top = store.similarity_search("otter river", k=2)
+        assert {d.id for d in top} == {"a1", "a2"}
+        assert top[0].metadata["parent"] == "doc-a"
+        # Metadata round-trips whole, including keys that are not fields.
+        got = store.get_by_ids(["a1"])[0]
+        assert got.metadata == chunks[0].metadata and got.page_content == chunks[0].page_content
+        # Filters on declared fields: a plain dict, or a Cairn filter.
+        only_b = store.similarity_search("otter river", k=4, filter={"source": "b"})
+        assert {d.id for d in only_b} == {"b1", "b2"}
+        pages = store.similarity_search("otter", k=4, filter={"field": "page", "gte": 2})
+        assert {d.id for d in pages} == {"a2", "b2"}
+        scored = store.similarity_search_with_relevance_scores("otter river", k=4)
+        assert all(0.0 <= s <= 1.0 for _, s in scored) and scored[0][1] >= scored[-1][1]
+        # One hit per parent, a retriever, and hybrid search.
+        grouped = store.similarity_search("otter river market", k=4, group_by="parent")
+        assert sorted(d.metadata["parent"] for d in grouped) == ["doc-a", "doc-b"]
+        assert store.as_retriever(search_kwargs={"k": 1}).invoke("stock market")[0].id == "b1"
+        assert store.hybrid_search("election", k=1)[0].id == "b2"
+
+        # Deletions are final, through any read.
+        store.delete(parent="doc-a")
+        assert {d.id for d in store.similarity_search("otter river", k=4)} == {"b1", "b2"}
+        assert store.get_by_ids(["a1", "a2"]) == []
+        store.delete(ids=["b1"])
+        assert [d.id for d in store.similarity_search("market", k=4)] == ["b2"]
+
+        again = CairnVectorStore.from_texts(["hello otter"], WordHash(), client=store.client, ids=["h1"])
+        assert again.similarity_search("otter", k=1)[0].id == "h1"
+    finally:
+        admin.drop_collection(name)
+    with pytest.raises(NotFoundError):
+        store.similarity_search("otter", k=1)