fenecdb 0.1.5__py3-none-any.whl

fenecdb/__init__.py

@@ -0,0 +1,82 @@
+"""fenecdb over HTTP.
+
+A client for fenec-pg's HTTP endpoint (`fenec-pg --http <address>`), and on
+top of it vector stores for LangChain (`fenecdb.langchain`) and LlamaIndex
+(`fenecdb.llama_index`).
+
+    from fenecdb import Client
+    db = Client("http://127.0.0.1:8080", token="...")
+    db.query("get articles near embed $1 limit 5", [[0.1, 0.2, 0.3]])
+
+The client is the standard library alone, as the server is: a statement is
+one `POST /query`, several are one `POST /batch` under one lock.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from typing import Any, Iterable, Sequence
+
+__all__ = ["Client", "FenecError", "placeholders"]
+
+
+class FenecError(Exception):
+    """A statement the server refused: its message, and the HTTP status."""
+
+    def __init__(self, message: str, status: int):
+        super().__init__(message)
+        self.status = status
+
+
+class Client:
+    """fenec-pg's HTTP endpoint, one statement a request."""
+
+    def __init__(
+        self,
+        url: str = "http://127.0.0.1:8080",
+        token: str | None = None,
+        timeout: float = 30.0,
+    ):
+        self.url = url.rstrip("/")
+        self.token = token
+        self.timeout = timeout
+
+    def query(self, fenecql: str, params: Sequence[Any] | None = None) -> Any:
+        """Runs one FenecQL statement. Values go in as `$1`, `$2`, ... and
+        never into the text."""
+        body = {"query": fenecql, "params": list(params or [])}
+        return self._post("/query", json.dumps(body).encode(), "application/json")
+
+    def batch(self, statements: Iterable[tuple[str, Sequence[Any]]]) -> Any:
+        """Runs statements in order under one write lock, as one block:
+        their writes -- a create, a drop or a `create index` among them --
+        all land or, at the first error, none of them do. A batch holding a
+        `compact` runs each statement on its own, and what ran before an
+        error stays."""
+        lines = [json.dumps({"query": q, "params": list(p)}) for q, p in statements]
+        return self._post("/batch", "\n".join(lines).encode(), "application/x-ndjson")
+
+    def _post(self, path: str, body: bytes, content_type: str) -> Any:
+        req = urllib.request.Request(self.url + path, data=body, method="POST")
+        req.add_header("Content-Type", content_type)
+        if self.token:
+            req.add_header("Authorization", f"Bearer {self.token}")
+        try:
+            with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                return json.load(resp)
+        except urllib.error.HTTPError as e:
+            raw = e.read()
+            try:
+                message = json.loads(raw).get("error", raw.decode(errors="replace"))
+            except (ValueError, AttributeError):
+                message = raw.decode(errors="replace")
+            raise FenecError(message, e.code) from None
+
+
+def placeholders(start: int, n: int) -> str:
+    """`$start, ..., $(start+n-1)`: a list literal's worth of parameters.
+    FenecQL binds a parameter to a value, not to a list, so `in` takes one
+    per element -- `in [$2, $3, $4]`."""
+    return ", ".join(f"${i}" for i in range(start, start + n))

fenecdb/langchain.py

@@ -0,0 +1,306 @@
+"""A LangChain vector store over fenecdb's HTTP endpoint.
+
+    from fenecdb.langchain import FenecVectorStore
+
+    store = FenecVectorStore(embeddings, "docs", url="http://127.0.0.1:8080", token="...")
+    store.add_documents(documents)
+    store.similarity_search("how do I compact", k=4)
+
+A document is a row: its id, its text, its metadata as JSON, and its vector
+under `@hnsw` -- over int8 or bit codes with `quant="int8"` or `"bit"`. The
+collection is created on the first write, when the embedding's size is known.
+A metadata field named in `metadata_fields` also gets a column of its own,
+under `@hash`, and `filter` takes equality over those -- `filter={"source":
+"handbook"}` is `where source = $2`, answered by the index, before the search
+ranks what passed.
+
+With `full_text=True` the text is indexed for BM25 as well (`@text`), and a
+search takes `mode`: `"text"` ranks by the words alone (`match`), with no
+embedding of the query, and `"hybrid"` ranks the words and the vector each to
+their own depth and fuses the two rankings (`match ... near ... fuse`). A
+retriever passes it through `search_kwargs={"mode": "hybrid"}`. The scores
+are then BM25's, or the fusion's, not similarities.
+
+Writing an id that is already there replaces it: the old row out and the new
+one in, sent as one batch and run under one write lock.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import uuid
+from typing import Any, Callable, Iterable, Sequence
+
+from langchain_core.documents import Document
+from langchain_core.embeddings import Embeddings
+from langchain_core.vectorstores import VectorStore
+
+from fenecdb import Client, FenecError, placeholders
+
+__all__ = ["FenecVectorStore"]
+
+# Names go into the statement's text, so they are held to FenecQL's.
+_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_TYPES = {"text", "int", "float", "bool", "timestamp"}
+_RESERVED = {"id", "lc_id", "content", "metadata", "embedding"}
+_METRICS = {"cosine", "l2", "dot"}
+_QUANT = {None, "int8", "bit"}
+_MODES = {"similarity", "text", "hybrid"}
+
+
+class FenecVectorStore(VectorStore):
+    """A collection in fenecdb, holding LangChain documents."""
+
+    def __init__(
+        self,
+        embedding: Embeddings,
+        collection: str = "langchain",
+        *,
+        url: str = "http://127.0.0.1:8080",
+        token: str | None = None,
+        client: Client | None = None,
+        metric: str = "cosine",
+        metadata_fields: dict[str, str] | None = None,
+        quant: str | None = None,
+        full_text: bool = False,
+    ):
+        if not _NAME.match(collection):
+            raise ValueError(f"not a collection name: {collection!r}")
+        if metric not in _METRICS:
+            raise ValueError(f"metric is one of {sorted(_METRICS)}, not {metric!r}")
+        if quant not in _QUANT:
+            raise ValueError(f"quant is None, 'int8' or 'bit', not {quant!r}")
+        if quant == "bit" and metric != "cosine":
+            raise ValueError("quant='bit' keeps signs, which only cosine can order by")
+        fields = dict(metadata_fields or {})
+        for name, ty in fields.items():
+            if not _NAME.match(name) or name in _RESERVED:
+                raise ValueError(f"not a metadata field this store can index: {name!r}")
+            if ty not in _TYPES:
+                raise ValueError(f"{name}: a field is one of {sorted(_TYPES)}, not {ty!r}")
+        self._embedding = embedding
+        self.collection = collection
+        self._client = client or Client(url, token)
+        self._metric = metric
+        self._fields = fields
+        self._quant = quant
+        self._full_text = full_text
+        self._dimension: int | None = None
+
+    @property
+    def embeddings(self) -> Embeddings:
+        return self._embedding
+
+    # ------------------------------------------------------------ writing
+
+    def add_texts(
+        self,
+        texts: Iterable[str],
+        metadatas: list[dict] | None = None,
+        *,
+        ids: list[str | None] | None = None,
+        **kwargs: Any,
+    ) -> list[str]:
+        texts = list(texts)
+        if not texts:
+            return []
+        metadatas = list(metadatas) if metadatas else [{} for _ in texts]
+        # A document that came without an id gets one, and so does each of
+        # the documents without one among some that have theirs.
+        ids = [i or str(uuid.uuid4()) for i in (ids or [None] * len(texts))]
+        # numpy's floats are not JSON's.
+        vectors = [[float(x) for x in v] for v in self._embedding.embed_documents(texts)]
+        self._ensure(len(vectors[0]))
+
+        docs, params = [], []
+        for doc_id, text, meta, vector in zip(ids, texts, metadatas, vectors):
+            values = [doc_id, text, json.dumps(meta, default=str), vector]
+            values += [meta.get(f) for f in self._fields]
+            names = ["lc_id", "content", "metadata", "embedding", *self._fields]
+            first = len(params) + 1
+            docs.append(
+                "{" + ", ".join(f"{n}: ${first + i}" for i, n in enumerate(names)) + "}"
+            )
+            params += values
+        self._client.batch(
+            [
+                (f"del {self.collection} where lc_id in [{placeholders(1, len(ids))}]", ids),
+                (f"put {self.collection} [{', '.join(docs)}]", params),
+            ]
+        )
+        return ids
+
+    def delete(self, ids: list[str] | None = None, **kwargs: Any) -> bool | None:
+        """Deletes these ids, or every document when `ids` is None."""
+        if ids is None:
+            statement, params = f"del {self.collection}", []
+        else:
+            params = list(ids)
+            if not params:
+                return True
+            statement = (
+                f"del {self.collection} where lc_id in [{placeholders(1, len(params))}]"
+            )
+        self._run(statement, params)
+        return True
+
+    def drop_collection(self) -> None:
+        """Drops the collection, documents and index together."""
+        self._client.query(f"drop collection if exists {self.collection}")
+        self._dimension = None
+
+    # ------------------------------------------------------------ reading
+
+    def get_by_ids(self, ids: Sequence[str], /) -> list[Document]:
+        ids = list(ids)
+        if not ids:
+            return []
+        rows = self._run(
+            f"get {self.collection} select lc_id, content, metadata "
+            f"where lc_id in [{placeholders(1, len(ids))}]",
+            ids,
+        )
+        return [self._document(r) for r in rows]
+
+    def similarity_search(
+        self, query: str, k: int = 4, filter: dict | None = None, **kwargs: Any
+    ) -> list[Document]:
+        """`mode` as `similarity_search_with_score` takes it."""
+        return [d for d, _ in self.similarity_search_with_score(query, k, filter, **kwargs)]
+
+    def similarity_search_with_score(
+        self,
+        query: str,
+        k: int = 4,
+        filter: dict | None = None,
+        *,
+        mode: str = "similarity",
+        **kwargs: Any,
+    ) -> list[tuple[Document, float]]:
+        """Documents with fenecdb's score: the cosine similarity under
+        `cosine`, higher is nearer; the distance under `l2`; the product
+        under `dot`. With `mode="text"` or `"hybrid"` (`full_text=True`)
+        BM25's score, or the fusion's."""
+        if mode not in _MODES:
+            raise ValueError(f"mode is one of {sorted(_MODES)}, not {mode!r}")
+        if mode == "similarity":
+            vector = self._embedding.embed_query(query)
+            return self.similarity_search_with_score_by_vector(vector, k, filter, **kwargs)
+        if not self._full_text:
+            raise ValueError(f"mode {mode!r} needs the store made with full_text=True")
+        if mode == "text":
+            rank, ahead = "match content $1", [query]
+        else:
+            vector = [float(x) for x in self._embedding.embed_query(query)]
+            rank, ahead = "match content $1 near embedding $2 fuse", [query, vector]
+        where, params = self._where(filter, first=len(ahead) + 1)
+        rows = self._run(
+            f"get {self.collection} select lc_id, content, metadata{where} "
+            f"{rank} limit {int(k)}",
+            [*ahead, *params],
+        )
+        return [(self._document(r), r["_score"]) for r in rows]
+
+    def similarity_search_by_vector(
+        self, embedding: list[float], k: int = 4, filter: dict | None = None, **kwargs: Any
+    ) -> list[Document]:
+        pairs = self.similarity_search_with_score_by_vector(embedding, k, filter, **kwargs)
+        return [d for d, _ in pairs]
+
+    def similarity_search_with_score_by_vector(
+        self, embedding: list[float], k: int = 4, filter: dict | None = None, **kwargs: Any
+    ) -> list[tuple[Document, float]]:
+        where, params = self._where(filter, first=2)
+        rows = self._run(
+            f"get {self.collection} select lc_id, content, metadata{where} "
+            f"near embedding $1 limit {int(k)}",
+            [[float(x) for x in embedding], *params],
+        )
+        return [(self._document(r), r["_score"]) for r in rows]
+
+    def _select_relevance_score_fn(self) -> Callable[[float], float]:
+        # Under `cosine` the score already is 1 - distance, LangChain's
+        # relevance; under `dot` the product is, for normalised vectors.
+        if self._metric == "l2":
+            return self._euclidean_relevance_score_fn
+        return lambda score: score
+
+    @classmethod
+    def from_texts(
+        cls,
+        texts: list[str],
+        embedding: Embeddings,
+        metadatas: list[dict] | None = None,
+        *,
+        ids: list[str] | None = None,
+        **kwargs: Any,
+    ) -> "FenecVectorStore":
+        store = cls(embedding, **kwargs)
+        store.add_texts(texts, metadatas, ids=ids)
+        return store
+
+    # ------------------------------------------------------------ inside
+
+    def _ensure(self, dimension: int) -> None:
+        if self._dimension == dimension:
+            return
+        extra = "".join(f", {f} {t} @hash" for f, t in self._fields.items())
+        quant = f", quant={self._quant}" if self._quant else ""
+        content = "content text @text" if self._full_text else "content text"
+        self._client.query(
+            f"create collection if not exists {self.collection} ("
+            f"lc_id text @hash, {content}, metadata text, "
+            f"embedding vector<{dimension}> @hnsw({self._metric}{quant}){extra})"
+        )
+        if self._full_text:
+            # A collection made before without it takes the index now.
+            self._client.query(
+                f"create index if not exists on {self.collection} (content) @text"
+            )
+        self._dimension = dimension
+
+    def _run(self, statement: str, params: list) -> Any:
+        """A statement over a collection that may not be there yet: a store
+        nothing was written to is an empty one, not an error."""
+        try:
+            return self._client.query(statement, params)
+        except FenecError as e:
+            if e.status == 404 and str(e).startswith("collection `"):
+                return []
+            raise
+
+    def _where(self, filter: dict | None, first: int) -> tuple[str, list]:
+        """` where f = $n and g in [$m, ...]` over the indexed fields."""
+        if not filter:
+            return "", []
+        terms, params = [], []
+        for name, want in filter.items():
+            if name not in self._fields:
+                raise ValueError(
+                    f"`{name}` is not in metadata_fields: only those can be filtered on"
+                )
+            if isinstance(want, dict):
+                if set(want) == {"$eq"}:
+                    want = want["$eq"]
+                elif set(want) == {"$in"}:
+                    want = list(want["$in"])
+                else:
+                    raise ValueError(f"{name}: only $eq and $in are supported, not {want}")
+            if isinstance(want, (list, tuple, set)):
+                values = list(want)
+                n = first + len(params)
+                terms.append(f"{name} in [{placeholders(n, len(values))}]")
+                params += values
+            else:
+                terms.append(f"{name} = ${first + len(params)}")
+                params.append(want)
+        return " where " + " and ".join(terms), params
+
+    @staticmethod
+    def _document(row: dict) -> Document:
+        return Document(
+            id=row["lc_id"],
+            page_content=row["content"],
+            metadata=json.loads(row["metadata"] or "{}"),
+        )

fenecdb/llama_index.py

@@ -0,0 +1,332 @@
+"""A LlamaIndex vector store over fenecdb's HTTP endpoint.
+
+    from fenecdb.llama_index import FenecVectorStore
+    from llama_index.core import StorageContext, VectorStoreIndex
+
+    store = FenecVectorStore("docs", url="http://127.0.0.1:8080", token="...")
+    index = VectorStoreIndex.from_documents(
+        documents, storage_context=StorageContext.from_defaults(vector_store=store)
+    )
+
+A node is a row: its id, its document's id, its text, its metadata as
+LlamaIndex serialises it, and its vector under `@hnsw` -- over int8 or bit
+codes with `quant="int8"` or `"bit"`. The collection is created on the first
+write, when the embedding's size is known. A metadata field named in
+`metadata_fields` also gets a column of its own, under `@hash`, and a query's
+filters are answered over those -- `==`, `!=`, `<`, `<=`, `>`, `>=`, `in` and
+`nin`, joined by `and` or `or` -- before the search ranks what passed.
+
+With `full_text=True` the text is indexed for BM25 as well (`@text`), and a
+query takes the modes that need it: `TEXT_SEARCH` ranks by the words alone
+(`match`), and `HYBRID` ranks the words and the vector each to their own depth
+and fuses the two rankings (`match ... near ... fuse`). The fusion adds ranks,
+not scores, so `alpha` has nothing to weigh and is not read; `hybrid_top_k`,
+or `sparse_top_k`, is how deep each side goes.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, List, Optional, Sequence
+
+from llama_index.core.bridge.pydantic import PrivateAttr
+from llama_index.core.schema import BaseNode, MetadataMode
+from llama_index.core.vector_stores.types import (
+    BasePydanticVectorStore,
+    FilterCondition,
+    FilterOperator,
+    MetadataFilter,
+    MetadataFilters,
+    VectorStoreQuery,
+    VectorStoreQueryMode,
+    VectorStoreQueryResult,
+)
+from llama_index.core.vector_stores.utils import (
+    metadata_dict_to_node,
+    node_to_metadata_dict,
+)
+
+from fenecdb import Client, FenecError, placeholders
+
+__all__ = ["FenecVectorStore"]
+
+_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_TYPES = {"text", "int", "float", "bool", "timestamp"}
+_RESERVED = {"id", "node_id", "ref_doc_id", "content", "metadata", "embedding"}
+_METRICS = {"cosine", "l2", "dot"}
+_QUANT = {None, "int8", "bit"}
+_WORDS = {VectorStoreQueryMode.TEXT_SEARCH, VectorStoreQueryMode.HYBRID}
+_COMPARE = {
+    FilterOperator.EQ: "=",
+    FilterOperator.NE: "!=",
+    FilterOperator.GT: ">",
+    FilterOperator.GTE: ">=",
+    FilterOperator.LT: "<",
+    FilterOperator.LTE: "<=",
+}
+
+
+class FenecVectorStore(BasePydanticVectorStore):
+    """A collection in fenecdb, holding LlamaIndex nodes."""
+
+    stores_text: bool = True
+    flat_metadata: bool = False
+
+    collection: str = "llama_index"
+    url: str = "http://127.0.0.1:8080"
+    token: Optional[str] = None
+    metric: str = "cosine"
+    metadata_fields: dict = {}
+    quant: Optional[str] = None
+    full_text: bool = False
+
+    _client: Client = PrivateAttr()
+    _dimension: Optional[int] = PrivateAttr(default=None)
+
+    def __init__(
+        self,
+        collection: str = "llama_index",
+        url: str = "http://127.0.0.1:8080",
+        token: Optional[str] = None,
+        metric: str = "cosine",
+        metadata_fields: Optional[dict] = None,
+        client: Optional[Client] = None,
+        quant: Optional[str] = None,
+        full_text: bool = False,
+        **kwargs: Any,
+    ):
+        if not _NAME.match(collection):
+            raise ValueError(f"not a collection name: {collection!r}")
+        if metric not in _METRICS:
+            raise ValueError(f"metric is one of {sorted(_METRICS)}, not {metric!r}")
+        if quant not in _QUANT:
+            raise ValueError(f"quant is None, 'int8' or 'bit', not {quant!r}")
+        if quant == "bit" and metric != "cosine":
+            raise ValueError("quant='bit' keeps signs, which only cosine can order by")
+        fields = dict(metadata_fields or {})
+        for name, ty in fields.items():
+            if not _NAME.match(name) or name in _RESERVED:
+                raise ValueError(f"not a metadata field this store can index: {name!r}")
+            if ty not in _TYPES:
+                raise ValueError(f"{name}: a field is one of {sorted(_TYPES)}, not {ty!r}")
+        super().__init__(
+            collection=collection,
+            url=url,
+            token=token,
+            metric=metric,
+            metadata_fields=fields,
+            quant=quant,
+            full_text=full_text,
+            **kwargs,
+        )
+        self._client = client or Client(url, token)
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "FenecVectorStore"
+
+    @property
+    def client(self) -> Client:
+        return self._client
+
+    # ------------------------------------------------------------ writing
+
+    def add(self, nodes: Sequence[BaseNode], **kwargs: Any) -> List[str]:
+        """Writes the nodes; a node id already there is replaced, the old
+        row out and the new one in as one batch under one write lock."""
+        nodes = list(nodes)
+        if not nodes:
+            return []
+        ids = [n.node_id for n in nodes]
+        vectors = [[float(x) for x in n.get_embedding()] for n in nodes]
+        self._ensure(len(vectors[0]))
+        names = ["node_id", "ref_doc_id", "content", "metadata", "embedding"]
+        names += list(self.metadata_fields)
+        docs, params = [], []
+        for node, vector in zip(nodes, vectors):
+            meta = node_to_metadata_dict(
+                node, remove_text=True, flat_metadata=self.flat_metadata
+            )
+            values = [
+                node.node_id,
+                node.ref_doc_id,
+                node.get_content(metadata_mode=MetadataMode.NONE),
+                json.dumps(meta, default=str),
+                vector,
+            ]
+            values += [node.metadata.get(f) for f in self.metadata_fields]
+            first = len(params) + 1
+            docs.append(
+                "{" + ", ".join(f"{n}: ${first + i}" for i, n in enumerate(names)) + "}"
+            )
+            params += values
+        c = self.collection
+        self._client.batch(
+            [
+                (f"del {c} where node_id in [{placeholders(1, len(ids))}]", ids),
+                (f"put {c} [{', '.join(docs)}]", params),
+            ]
+        )
+        return ids
+
+    def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None:
+        """Deletes every node of a document."""
+        self._run(f"del {self.collection} where ref_doc_id = $1", [ref_doc_id])
+
+    def delete_nodes(
+        self,
+        node_ids: Optional[List[str]] = None,
+        filters: Optional[MetadataFilters] = None,
+        **delete_kwargs: Any,
+    ) -> None:
+        """Deletes these nodes, or those the filters pass. Given neither it
+        deletes nothing: everything is `clear`, and asked for by name."""
+        if node_ids is None and filters is None:
+            return
+        where, params = self._where(node_ids, None, filters, first=1)
+        self._run(f"del {self.collection}{where}", params)
+
+    def clear(self) -> None:
+        self._run(f"del {self.collection}", [])
+
+    def drop_collection(self) -> None:
+        """Drops the collection, nodes and index together."""
+        self._client.query(f"drop collection if exists {self.collection}")
+        self._dimension = None
+
+    # ------------------------------------------------------------ reading
+
+    def get_nodes(
+        self,
+        node_ids: Optional[List[str]] = None,
+        filters: Optional[MetadataFilters] = None,
+    ) -> List[BaseNode]:
+        where, params = self._where(node_ids, None, filters, first=1)
+        rows = self._run(
+            f"get {self.collection} select node_id, content, metadata{where}", params
+        )
+        return [self._node(r) for r in rows]
+
+    def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult:
+        mode = query.mode
+        if mode not in (VectorStoreQueryMode.DEFAULT, *_WORDS):
+            raise NotImplementedError(f"query mode {mode} is not supported")
+        if mode in _WORDS and not self.full_text:
+            raise ValueError(f"query mode {mode} needs the store made with full_text=True")
+        if mode in _WORDS and not query.query_str:
+            raise ValueError(f"query mode {mode} needs the query's text")
+        if mode != VectorStoreQueryMode.TEXT_SEARCH and query.query_embedding is None:
+            raise ValueError("a query embedding is needed")
+        # What the query ranks by, and its parameters ahead of the filter's.
+        rank, ahead = self._rank(query)
+        # An empty list is no restriction here, as LlamaIndex means it: its
+        # retriever sends `node_ids=[]` for a store that keeps its own text.
+        where, params = self._where(
+            query.node_ids or None, query.doc_ids or None, query.filters, first=len(ahead) + 1
+        )
+        rows = self._run(
+            f"get {self.collection} select node_id, content, metadata{where} "
+            f"{rank} limit {int(query.similarity_top_k)}",
+            [*ahead, *params],
+        )
+        return VectorStoreQueryResult(
+            nodes=[self._node(r) for r in rows],
+            similarities=[r["_score"] for r in rows],
+            ids=[r["node_id"] for r in rows],
+        )
+
+    # ------------------------------------------------------------ inside
+
+    def _rank(self, query: VectorStoreQuery) -> tuple[str, list]:
+        """`near`, `match`, or both fused, and their parameters, $1 on."""
+        vector = [float(x) for x in query.query_embedding or []]
+        if query.mode == VectorStoreQueryMode.TEXT_SEARCH:
+            return "match content $1", [query.query_str]
+        if query.mode == VectorStoreQueryMode.HYBRID:
+            depth = query.hybrid_top_k or query.sparse_top_k
+            deep = f" candidates {int(depth)}" if depth else ""
+            return f"match content $1 near embedding $2 fuse{deep}", [query.query_str, vector]
+        return "near embedding $1", [vector]
+
+    def _ensure(self, dimension: int) -> None:
+        if self._dimension == dimension:
+            return
+        extra = "".join(f", {f} {t} @hash" for f, t in self.metadata_fields.items())
+        quant = f", quant={self.quant}" if self.quant else ""
+        content = "content text @text" if self.full_text else "content text"
+        self._client.query(
+            f"create collection if not exists {self.collection} ("
+            f"node_id text @hash, ref_doc_id text @hash, {content}, metadata text, "
+            f"embedding vector<{dimension}> @hnsw({self.metric}{quant}){extra})"
+        )
+        if self.full_text:
+            # A collection made before without it takes the index now.
+            self._client.query(
+                f"create index if not exists on {self.collection} (content) @text"
+            )
+        self._dimension = dimension
+
+    def _run(self, statement: str, params: list) -> Any:
+        """A collection nothing was written to yet is an empty one."""
+        try:
+            return self._client.query(statement, params)
+        except FenecError as e:
+            if e.status == 404 and str(e).startswith("collection `"):
+                return []
+            raise
+
+    def _where(
+        self,
+        node_ids: Optional[List[str]],
+        doc_ids: Optional[List[str]],
+        filters: Optional[MetadataFilters],
+        first: int,
+    ) -> tuple[str, list]:
+        terms, params = [], []
+        for field, values in (("node_id", node_ids), ("ref_doc_id", doc_ids)):
+            if values is not None:
+                n = first + len(params)
+                terms.append(f"{field} in [{placeholders(n, len(values))}]")
+                params += list(values)
+        if filters is not None and filters.filters:
+            terms.append(self._filters(filters, params, first))
+        if not terms:
+            return "", []
+        return " where " + " and ".join(terms), params
+
+    def _filters(self, filters: MetadataFilters, params: list, first: int) -> str:
+        joiner = {FilterCondition.AND: " and ", FilterCondition.OR: " or "}.get(
+            filters.condition
+        )
+        if joiner is None:
+            raise NotImplementedError(f"filter condition {filters.condition} is not supported")
+        parts = []
+        for f in filters.filters:
+            if isinstance(f, MetadataFilters):
+                parts.append(self._filters(f, params, first))
+            else:
+                parts.append(self._filter(f, params, first))
+        return "(" + joiner.join(parts) + ")"
+
+    def _filter(self, f: MetadataFilter, params: list, first: int) -> str:
+        if f.key not in self.metadata_fields:
+            raise ValueError(
+                f"`{f.key}` is not in metadata_fields: only those can be filtered on"
+            )
+        if f.operator in _COMPARE:
+            params.append(f.value)
+            return f"{f.key} {_COMPARE[f.operator]} ${first + len(params) - 1}"
+        if f.operator in (FilterOperator.IN, FilterOperator.NIN):
+            values = list(f.value)
+            n = first + len(params)
+            params += values
+            term = f"{f.key} in [{placeholders(n, len(values))}]"
+            return term if f.operator == FilterOperator.IN else f"not ({term})"
+        raise NotImplementedError(f"filter operator {f.operator} is not supported")
+
+    @staticmethod
+    def _node(row: dict) -> BaseNode:
+        node = metadata_dict_to_node(json.loads(row["metadata"] or "{}"))
+        node.set_content(row["content"] or "")
+        return node

fenecdb-0.1.5.dist-info/METADATA

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: fenecdb
+Version: 0.1.5
+Summary: fenecdb over HTTP: a client, and vector stores for LangChain and LlamaIndex
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://fenecdb.com
+Project-URL: Documentation, https://fenecdb.com/docs/integrations
+Project-URL: Source, https://github.com/fenecdb/fenec
+Keywords: fenecdb,vector,database,langchain,llama-index
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=1.0; extra == "langchain"
+Provides-Extra: llama-index
+Requires-Dist: llama-index-core>=0.12; extra == "llama-index"
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+Requires-Dist: pytest-asyncio>=0.23; extra == "test"
+Requires-Dist: langchain-core>=1.0; extra == "test"
+Requires-Dist: langchain-tests>=1.0; extra == "test"
+Requires-Dist: llama-index-core>=0.12; extra == "test"
+
+# fenecdb for Python
+
+A client for fenec-pg's HTTP endpoint (`fenec-pg --http <address>`), and
+vector stores for LangChain and LlamaIndex on top of it.
+
+```sh
+pip install "fenecdb[langchain] @ git+https://github.com/fenecdb/fenec#subdirectory=integrations/python"
+# or fenecdb[llama-index]
+```
+
+```python
+from fenecdb import Client
+
+db = Client("http://127.0.0.1:8080", token="...")
+db.query("get articles select title near embed $1 limit 5", [[0.1, 0.2, 0.3]])
+```
+
+```python
+from fenecdb.langchain import FenecVectorStore
+
+store = FenecVectorStore(embeddings, "docs", url="http://127.0.0.1:8080", token="...",
+                         metadata_fields={"source": "text"})
+store.add_documents(documents)
+store.similarity_search("how do I compact", k=4, filter={"source": "handbook"})
+```
+
+```python
+from fenecdb.llama_index import FenecVectorStore
+from llama_index.core import StorageContext, VectorStoreIndex
+
+store = FenecVectorStore("docs", url="http://127.0.0.1:8080", token="...")
+index = VectorStoreIndex.from_documents(
+    documents, storage_context=StorageContext.from_defaults(vector_store=store)
+)
+```
+
+A document or node is a row -- its id, text, metadata as JSON and vector
+under `@hnsw` -- in a collection created on the first write. Fields named in
+`metadata_fields` get columns of their own under `@hash`, and filters over
+them are answered by the index before the search runs.
+
+With `full_text=True` the text is indexed for BM25 too, and both stores search
+by the words alone or by the words and the vector fused -- LangChain's
+`mode="text"` and `mode="hybrid"`, LlamaIndex's `TEXT_SEARCH` and `HYBRID`.
+`quant="int8"` or `"bit"` keeps the graph over codes.
+
+```python
+store = FenecVectorStore(embeddings, "docs", url=..., token=..., full_text=True)
+store.similarity_search("how do I compact", k=4, mode="hybrid")
+```
+
+`./run-tests.sh` runs LangChain's standard vector store suite and the tests
+LlamaIndex's own integrations run against a fenec-pg it builds and starts.
+Full reference: https://fenecdb.com/docs/integrations

fenecdb-0.1.5.dist-info/RECORD

@@ -0,0 +1,7 @@
+fenecdb/__init__.py,sha256=mMrd365ZJfduOH8fsNi6v7UK5txkcEKjvy07zr1-V_4,3188
+fenecdb/langchain.py,sha256=ncPhxU2Y8L1iHi9MwMxlWNWZyyv_HmCYFKcwN1woblE,12520
+fenecdb/llama_index.py,sha256=KnY5h43SABEw0ZbIE_hCJ3qcqzoEVAxF5rZSJsL2cMY,13532
+fenecdb-0.1.5.dist-info/METADATA,sha256=OTBTlazouURidyjdj2l9IVcEgfba_9HvTAJjYnyACx8,2941
+fenecdb-0.1.5.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+fenecdb-0.1.5.dist-info/top_level.txt,sha256=Qr23SMzjRRS9xJQGoT06bXtReVRfrjhjuMO3vEclBCA,8
+fenecdb-0.1.5.dist-info/RECORD,,

fenecdb-0.1.5.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fenecdb-0.1.5.dist-info/top_level.txt

@@ -0,0 +1 @@
+fenecdb