langchain-envector 0.1.0__py3-none-any.whl

langchain_envector/__init__.py

@@ -0,0 +1,10 @@
+"""Envector LangChain integration package.
+
+Provides a LangChain-compatible VectorStore that wraps the high-level `es2` SDK.
+All code and comments are in English as per project rules.
+"""
+
+from .vectorstore import Envector
+
+__all__ = ["Envector"]
+

langchain_envector/client.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from .config import EnvectorConfig
+
+
+class EnvectorClient:
+    """Thin convenience client around the high-level `es2` SDK.
+
+    - Establishes a connection
+    - Initializes key and index configuration
+    - Optionally creates the index if missing
+    - Provides access to the ES2 `Index` instance
+    """
+
+    def __init__(self, config: EnvectorConfig):
+        self.config = config
+        self._es2 = None
+        self._index = None
+
+    def init(self):
+        import es2
+
+        c = self.config.connection
+        k = self.config.key
+        i = self.config.index
+
+        es2_client = es2.ES2()
+
+        # Connection
+        if c.address:
+            es2_client.init_connect(address=c.address, access_token=c.access_token)
+        else:
+            if not (c.host and c.port):
+                raise ValueError("Either address or host+port must be provided.")
+            es2_client.init_connect(host=c.host, port=c.port, access_token=c.access_token)
+
+        # Key path baseline for Index
+        from es2.index import Index as _Index
+
+        _Index.init_key_path(k.key_path)
+
+        # Index config + key setup
+        es2_client.init_index_config(
+            index_name=i.index_name,
+            dim=i.dim,
+            key_path=k.key_path,
+            key_id=k.key_id,
+            seal_mode=k.seal_mode,
+            seal_kek_path=k.seal_kek_path,
+            preset=k.preset,
+            eval_mode=k.eval_mode,
+            query_encryption=i.query_encryption,
+            index_encryption="cipher",  # server vectors are always encrypted
+            index_type=i.index_type,
+            auto_key_setup=True,
+        )
+
+        # Create index if missing
+        if self.config.create_if_missing:
+            idx_list = es2_client.get_index_list()
+            if i.index_name not in idx_list:
+                es2_client.create_index(index_name=i.index_name, dim=i.dim)
+
+        # Bind index instance
+        self._index = es2.Index(i.index_name)
+        self._es2 = es2_client
+        return self
+
+    @property
+    def index(self):
+        if self._index is None:
+            raise RuntimeError("Client not initialized. Call init().")
+        return self._index
+
+    @property
+    def es2(self):
+        if self._es2 is None:
+            raise RuntimeError("Client not initialized. Call init().")
+        return self._es2
+

langchain_envector/config.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+
+@dataclass
+class ConnectionConfig:
+    address: Optional[str] = None
+    host: Optional[str] = None
+    port: Optional[int] = None
+    access_token: Optional[str] = None
+
+
+@dataclass
+class KeyConfig:
+    key_path: str
+    key_id: str
+    preset: Optional[str] = None
+    eval_mode: Optional[str] = None
+    seal_mode: Optional[str] = None
+    seal_kek_path: Optional[str] = None
+
+
+@dataclass
+class IndexSettings:
+    index_name: str
+    dim: int
+    query_encryption: str = "plain"  # plain | cipher
+    index_encryption: str = "cipher"  # fixed to cipher
+    index_type: str = "flat"
+    output_fields: List[str] = field(default_factory=lambda: ["metadata"])
+    fetch_k: Optional[int] = None  # over-fetch to support client-side filters
+
+
+@dataclass
+class EnvectorConfig:
+    connection: ConnectionConfig
+    key: KeyConfig
+    index: IndexSettings
+    create_if_missing: bool = True
+

langchain_envector/retriever.py

@@ -0,0 +1,20 @@
+"""Optional custom retriever wrapper.
+
+Prefer using VectorStore.as_retriever, which returns LangChain's VectorStoreRetriever
+when available. This module exists as an extension point if custom behavior is needed.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from .vectorstore import Envector, Document
+
+
+class EnvectorRetriever:
+    def __init__(self, store: Envector, *, search_kwargs: Optional[Dict[str, Any]] = None) -> None:
+        self.store = store
+        self.search_kwargs = search_kwargs or {}
+
+    def get_relevant_documents(self, query: str) -> List[Document]:
+        return self.store.similarity_search(query, **self.search_kwargs)

langchain_envector/types.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Protocol, Tuple, Union, overload
+
+
+class Embeddings(Protocol):
+    """Minimal Embeddings protocol to avoid hard-coding a dependency on LangChain.
+
+    LangChain-compatible embeddings typically implement these two methods.
+    """
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:  # pragma: no cover - interface only
+        ...
+
+    def embed_query(self, text: str) -> List[float]:  # pragma: no cover - interface only
+        ...
+
+
+@dataclass
+class SearchResult:
+    id: str
+    score: float
+    metadata: Dict[str, Any]
+
+
+def pack_metadata(text: str, metadata: Optional[Dict[str, Any]] = None) -> str:
+    """Pack text and metadata into a single JSON string field accepted by ES2.
+
+    ES2 metadata API stores lists of strings; we store a single JSON blob per item.
+    Item-level IDs are not persisted/addressable.
+    """
+    import json
+
+    payload = {
+        "text": text,
+        "metadata": metadata or {},
+    }
+    return json.dumps(payload, ensure_ascii=False)
+
+
+def unpack_metadata(raw: Any) -> Dict[str, Any]:
+    """Return metadata as a dict regardless of the raw payload type.
+
+    Recent ES2 versions may return decrypted metadata as a Python dict instead
+    of the JSON string we originally stored. We normalise the payload here so
+    downstream code always works with a dictionary.
+    """
+
+    import json
+
+    # Already a dict → nothing to do.
+    if isinstance(raw, dict):
+        return raw
+
+    print("slafjklshglkhslafhlksadjlghsal;hf")
+
+    # Some responses wrap the payload in a single-element list.
+    if isinstance(raw, (list, tuple)):
+        if len(raw) == 1:
+            return unpack_metadata(raw[0])
+        return {"_raw": list(raw)}
+
+    if raw is None:
+        return {"_raw": None}
+
+    # Decode bytes before attempting JSON parsing.
+    if isinstance(raw, (bytes, bytearray)):
+        try:
+            raw = raw.decode("utf-8")
+        except Exception:
+            return {"_raw": raw}
+
+    if isinstance(raw, str):
+        try:
+            data = json.loads(raw)
+            if isinstance(data, dict):
+                return data
+        except Exception:
+            # Some ES2 responses return Python-literal strings (single quotes).
+            try:
+                import ast
+
+                data = ast.literal_eval(raw)
+                if isinstance(data, dict):
+                    return data
+            except Exception:
+                pass
+        return {"_raw": raw}
+
+    # Fallback: expose the raw object for debugging purposes.
+    return {"_raw": raw}
+
+
+# --- Embeddings adaptation helpers -----------------------------------------------------
+
+class _CallableEmbeddings:
+    def __init__(self, docs_fn: Callable[[List[str]], List[List[float]]], query_fn: Callable[[str], List[float]]):
+        self._docs_fn = docs_fn
+        self._query_fn = query_fn
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        return self._docs_fn(texts)
+
+    def embed_query(self, text: str) -> List[float]:
+        return self._query_fn(text)
+
+
+def as_embeddings(emb: Any) -> Embeddings:
+    """Adapt various embedding providers into the Embeddings protocol.
+
+    Supported inputs:
+    - Object with `embed_documents` and `embed_query` (LangChain-style) — returned as-is.
+    - Object with `encode` supporting list[str] -> list[vec] and str -> vec — wrapped.
+    - Tuple(callable_docs, callable_query) — wrapped.
+    """
+    # Case 1: Already satisfies the protocol
+    if hasattr(emb, "embed_documents") and hasattr(emb, "embed_query"):
+        return emb  # type: ignore[return-value]
+
+    # Case 2: Sentence-Transformers style `.encode()`
+    if hasattr(emb, "encode"):
+        encode = getattr(emb, "encode")
+
+        def docs_fn(texts: List[str]) -> List[List[float]]:
+            return list(encode(texts))
+
+        def query_fn(text: str) -> List[float]:
+            out = encode([text])
+            return list(out[0]) if isinstance(out, (list, tuple)) else list(out)
+
+        return _CallableEmbeddings(docs_fn, query_fn)
+
+    # Case 3: Tuple of callables
+    if isinstance(emb, tuple) and len(emb) == 2 and callable(emb[0]) and callable(emb[1]):
+        docs_fn, query_fn = emb  # type: ignore[assignment]
+        return _CallableEmbeddings(docs_fn, query_fn)
+
+    raise TypeError(
+        "Unsupported embeddings provider. Provide an object with 'embed_documents'/'embed_query', "
+        "an object with 'encode', or a tuple of two callables (docs_fn, query_fn)."
+    )

langchain_envector/vectorstore.py

@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, Iterable, List, Optional, Sequence
+from uuid import uuid4
+
+from .config import EnvectorConfig
+from .client import EnvectorClient
+from .types import Embeddings, as_embeddings, pack_metadata, unpack_metadata
+
+
+def _try_import_langchain():
+    """Return (VectorStoreBase, DocumentClass) with safe fallbacks.
+
+    Ensures we always return a valid base class even if LangChain is missing.
+    """
+    VectorStoreBase: Any = object
+
+    try:
+        from langchain_core.documents import Document  # type: ignore
+    except Exception:  # pragma: no cover - optional dependency
+        # Minimal shim if LangChain is not installed
+        class Document:  # type: ignore
+            def __init__(self, page_content: str, metadata: Optional[Dict[str, Any]] = None):
+                self.page_content = page_content
+                self.metadata = metadata or {}
+
+    try:
+        from langchain_core.vectorstores import VectorStore as _VectorStore  # type: ignore
+        VectorStoreBase = _VectorStore
+    except Exception:  # pragma: no cover - optional dependency
+        pass
+
+    return VectorStoreBase, Document
+
+
+VectorStore, Document = _try_import_langchain()
+
+
+class Envector(VectorStore):  # type: ignore[misc]
+    """LangChain-compatible VectorStore adaptor for Envector (ES2).
+
+    This class wraps the high-level `es2` SDK. It does not use low-level
+    gRPC stubs or `es2.api.Indexer` directly.
+    """
+
+    def __init__(
+        self,
+        *,
+        config: EnvectorConfig,
+        embeddings: Optional[Embeddings] = None,
+        client: Optional[EnvectorClient] = None,
+    ) -> None:
+        self.config = config
+        self._embeddings = as_embeddings(embeddings) if embeddings is not None else None
+        self.client = client or EnvectorClient(config)
+        self.client.init()
+
+    # -------------------------------
+    # VectorStore API
+    # -------------------------------
+    def add_texts(
+        self,
+        texts: List[str],
+        metadatas: Optional[List[Dict[str, Any]]] = None,
+        ids: Optional[List[str]] = None,
+        *,
+        vectors: Optional[List[List[float]]] = None,
+        **kwargs: Any,
+    ) -> List[str]:
+        """Add texts to the encrypted index.
+
+        If embeddings are provided, the texts are embedded automatically.
+        Otherwise, provide pre-computed `vectors`.
+        """
+        if not texts:
+            return []
+
+        if metadatas is None:
+            metadatas = [{} for _ in texts]
+        if len(metadatas) != len(texts):
+            raise ValueError("texts and metadatas must have equal length")
+
+        if vectors is None:
+            if self._embeddings is None:
+                raise ValueError("embeddings is None and vectors not provided")
+            vectors = self._embeddings.embed_documents(texts)
+
+        # Prepare metadata JSON strings per item
+        packed = [pack_metadata(t, m) for t, m in zip(texts, metadatas)]
+
+        # Insert using high-level ES2 Index
+        self.client.index.insert(data=vectors, metadata=packed)
+
+        # Return ephemeral placeholders to satisfy VectorStore interface,
+        # but they are NOT persisted/addressable.
+        return [f"ephemeral-{uuid4()}" for _ in texts]
+
+    def similarity_search(
+        self,
+        query: str,
+        k: int = 4,
+        *,
+        filter: Optional[Dict[str, Any]] = None,
+        score_threshold: Optional[float] = None,
+        fetch_k: Optional[int] = None,
+        **kwargs: Any,
+    ) -> List[Document]:
+        """Search similar items for a text query.
+
+        - Embeds query if embeddings are provided; else expect `embedding` kwarg.
+        - Applies optional client-side filter and score threshold.
+        """
+        embedding: Optional[List[float]] = kwargs.get("embedding")
+        if embedding is None:
+            if self._embeddings is None:
+                raise ValueError("embeddings is None and no `embedding` provided")
+            embedding = self._embeddings.embed_query(query)
+
+        top_k = fetch_k or self.config.index.fetch_k or k
+
+        results = self.client.index.search(query=embedding, top_k=top_k, output_fields=self.config.index.output_fields)
+        # ES2 Index.search returns a list for each query; we passed single query
+        result = results[0] if isinstance(results, list) and results and isinstance(results[0], list) else results
+
+        docs = []
+        # Iterate from top-1 to top-k
+        for item in result:
+            # item = {"id": ..., "score": float, "metadata": [str] or {...}}
+            score = float(item.get("score", 0.0))
+            md_obj_raw = item.get("metadata")
+        
+            # Metadata encryption/decryption is handled by the SDK.
+            # Envector currently supports a single associated data field (string).
+            # Convention: if the string is JSON like {"text": str, "metadata": {...}},
+            # we unpack it; otherwise, we treat the raw string as the document text.
+            md_obj = unpack_metadata(md_obj_raw)
+
+            text = md_obj.get("text", "") if "_raw" not in md_obj else md_obj["_raw"]
+            metadata = md_obj.get("metadata", {}) if "_raw" not in md_obj else {}
+
+            # client-side filter
+            if filter:
+                # simple dict-equality filter on top-level user metadata
+                matched = all(metadata.get(k) == v for k, v in filter.items())
+                if not matched:
+                    continue
+            if score_threshold is not None and score < score_threshold:
+                continue
+
+            doc = Document(page_content=text, metadata={**metadata, "_score": score, "_id": item.get("id")})
+            docs.append(doc)
+
+        # Trim to k after filtering
+        return docs[:k]
+
+    # Vector-based variant required by some VectorStore interfaces
+    def similarity_search_by_vector(
+        self,
+        embedding: List[float],
+        k: int = 4,
+        *,
+        filter: Optional[Dict[str, Any]] = None,
+        score_threshold: Optional[float] = None,
+        fetch_k: Optional[int] = None,
+        **kwargs: Any,
+    ) -> List[Document]:
+        return self.similarity_search(
+            query="",  # unused
+            k=k,
+            filter=filter,
+            score_threshold=score_threshold,
+            fetch_k=fetch_k,
+            embedding=embedding,
+            **kwargs,
+        )
+
+    # -------------------------------
+    # Class constructors (LangChain compatibility)
+    # -------------------------------
+    @classmethod
+    def from_texts(
+        cls,
+        texts: List[str],
+        metadatas: Optional[List[Dict[str, Any]]] = None,
+        *,
+        embeddings: Optional[Embeddings] = None,
+        ids: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> "Envector":  # type: ignore[override]
+        """Create a store from texts. Requires `config` in kwargs.
+
+        Example:
+            Envector.from_texts(texts, metadatas=..., embeddings=..., config=cfg)
+        """
+        config: Optional[EnvectorConfig] = kwargs.get("config")  # type: ignore
+        client: Optional[EnvectorClient] = kwargs.get("client")  # type: ignore
+        if config is None:
+            raise ValueError("`config` (EnvectorConfig) is required for from_texts().")
+        store = cls(config=config, embeddings=embeddings, client=client)
+        store.add_texts(texts=texts, metadatas=metadatas, ids=ids)
+        return store
+
+    @classmethod
+    def from_documents(
+        cls,
+        documents: List[Document],
+        *,
+        embeddings: Optional[Embeddings] = None,
+        **kwargs: Any,
+    ) -> "Envector":  # type: ignore[override]
+        texts = [d.page_content for d in documents]
+        metadatas = [getattr(d, "metadata", {}) for d in documents]
+        return cls.from_texts(texts=texts, metadatas=metadatas, embeddings=embeddings, **kwargs)
+
+    # Optional: if LangChain is installed, this will be used; otherwise, users may call similarity_search directly.
+    def as_retriever(self, **kwargs: Any):  # pragma: no cover - wrapper
+        try:
+            from langchain_core.vectorstores import VectorStoreRetriever  # type: ignore
+
+            return VectorStoreRetriever(vectorstore=self, **kwargs)
+        except Exception:
+            # Minimal shim if VectorStoreRetriever is unavailable
+            class _Retriever:
+                def __init__(self, vs: Envector, search_kwargs: Optional[Dict[str, Any]] = None):
+                    self.vs = vs
+                    self.search_kwargs = search_kwargs or {}
+
+                def get_relevant_documents(self, query: str) -> List[Document]:
+                    return self.vs.similarity_search(query, **self.search_kwargs)
+
+            return _Retriever(self, kwargs.get("search_kwargs"))

langchain_envector-0.1.0.dist-info/METADATA

@@ -0,0 +1,73 @@
+Metadata-Version: 2.4
+Name: langchain-envector
+Version: 0.1.0
+Summary: LangChain VectorStore integration for Envector (ES2) encrypted vector search
+Author: Envector Contributors
+License: MIT
+Project-URL: Homepage, https://example.com/
+Project-URL: Repository, https://example.com/repo
+Keywords: langchain,vectorstore,homomorphic-encryption,ckks,encrypted-search,envector,es2
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: <3.14,>=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: es2==1.1.0rc1
+Requires-Dist: langchain>=0.2.0
+
+# LangChain Envector Integration
+
+Encrypted vector search for LangChain using Envector (ES2), powered by homomorphic encryption (CKKS). This repo ships a LangChain-compatible VectorStore and retriever utilities built on the high-level `es2` Python SDK.
+
+## Features
+- LangChain `VectorStore` interface with `similarity_search`, `from_texts`, etc.
+- Optional `VectorStoreRetriever` helper for quick RAG integrations.
+- Client-side encryption handled transparently by the SDK, including score thresholds and filtering.
+
+## Installation
+- Python 3.9–3.13 (recommend 3.11)
+- Create and activate a virtualenv:
+  - `python3.11 -m venv .venv && source .venv/bin/activate`
+- Install runtime dependencies:
+  - `pip install -U pip setuptools wheel`
+  - `pip install es2==1.1.0rc1 langchain sentence-transformers`
+
+## Usage Overview
+1. Configure Envector using `EnvectorConfig`, pointing to your ES2 endpoint and keys.
+2. Initialize embeddings (or provide pre-computed vectors).
+3. Instantiate `Envector(config=cfg, embeddings=emb)` and call `add_texts` or `as_retriever`.
+4. Run `similarity_search` or plug the retriever into your LangChain pipeline.
+
+> See `notebooks/` for end-to-end walkthroughs and the `libs/envector` package for implementation details.
+
+## Configuration
+Key dataclasses live in `libs/envector/config.py`:
+- `ConnectionConfig`: address or host/port for ES2.
+- `KeyConfig`: key path, key ID, optional preset/eval mode.
+- `IndexSettings`: index name, dimension (16–4096), query encryption mode, optional output fields and fetch parameters.
+- `EnvectorConfig`: wraps the above and enables auto-creation via `create_if_missing`.
+
+## Data Model
+- Each vector stores a single `metadata` string in ES2.
+- To align with LangChain’s `Document`, inserts wrap data as JSON: `{"text": ..., "metadata": ...}`.
+- Retrieval unwraps JSON, returning `Document(page_content=text, metadata={...})`.
+- Client-side filtering requires the JSON envelope to include an object under `metadata`.
+
+## Limitations
+- Item-level delete/update is unsupported (drop the index to reset).
+- Manual item IDs are not accepted; returned IDs from `add_texts` are ephemeral.
+- Filtering happens client-side; ensure metadata is JSON for structured filters.
+
+## Troubleshooting
+- Connection issues: verify ES2 address and registered keys.
+- Embeddings mismatch: ensure embedding dimension equals `index.dim` when supplying vectors.
+- Unexpected raw strings: confirm inserts used the JSON envelope.
+
+## Contributing
+See [`CONTRIBUTE.md`](CONTRIBUTE.md) for development, testing, and PR guidelines.

langchain_envector-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+langchain_envector/__init__.py,sha256=9th8ylpoi-hopUC5cXtJtfH_ycol6_N5X82_NwPcltA,248
+langchain_envector/client.py,sha256=8Y8CYBDnQ6-KMTat7q4qf94AXWi-SI_YeaqshgtajnI,2397
+langchain_envector/config.py,sha256=d67OgSqYEQXKJIGI0PFxgAkiauTJ3G5UN29bWJZ7tCc,1000
+langchain_envector/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+langchain_envector/retriever.py,sha256=Hx2OcyzTHSAGBRmBvnoddHtuXrqS0VTWGe2Q4JgxpVw,688
+langchain_envector/types.py,sha256=HaedNy-MquIw4Zmyy1wgGWPVge3LjbxpwCRxAEpiJHk,4607
+langchain_envector/vectorstore.py,sha256=hnxRBafefzmqBk10b_UdlD8DQ59JC3LQSpD5i3-d8ig,8813
+langchain_envector-0.1.0.dist-info/METADATA,sha256=YNng8JP4Oxync2cITzF9CtEUHw-XlMvSh_HOTM-XPAc,3544
+langchain_envector-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+langchain_envector-0.1.0.dist-info/top_level.txt,sha256=z87JZVH090LT8D8iNhPrBrTr94spsou5YscMWzTelYo,19
+langchain_envector-0.1.0.dist-info/RECORD,,

langchain_envector-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

langchain_envector-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+langchain_envector