lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/vectorstores/elasticsearch.py

@@ -0,0 +1,267 @@
+from typing import Any
+
+from elasticsearch import Elasticsearch
+from langchain.schema import Document
+from langchain_elasticsearch import ElasticsearchStore
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.io import (
+    BoolInput,
+    DropdownInput,
+    FloatInput,
+    HandleInput,
+    IntInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class ElasticsearchVectorStoreComponent(LCVectorStoreComponent):
+    """Elasticsearch Vector Store with with advanced, customizable search capabilities."""
+
+    display_name: str = "Elasticsearch"
+    description: str = "Elasticsearch Vector Store with with advanced, customizable search capabilities."
+    name = "Elasticsearch"
+    icon = "ElasticsearchStore"
+
+    inputs = [
+        StrInput(
+            name="elasticsearch_url",
+            display_name="Elasticsearch URL",
+            value="http://localhost:9200",
+            info="URL for self-managed Elasticsearch deployments (e.g., http://localhost:9200). "
+            "Do not use with Elastic Cloud deployments, use Elastic Cloud ID instead.",
+        ),
+        SecretStrInput(
+            name="cloud_id",
+            display_name="Elastic Cloud ID",
+            value="",
+            info="Use this for Elastic Cloud deployments. Do not use together with 'Elasticsearch URL'.",
+        ),
+        StrInput(
+            name="index_name",
+            display_name="Index Name",
+            value="langflow",
+            info="The index name where the vectors will be stored in Elasticsearch cluster.",
+        ),
+        *LCVectorStoreComponent.inputs,
+        StrInput(
+            name="username",
+            display_name="Username",
+            value="",
+            advanced=False,
+            info=(
+                "Elasticsearch username (e.g., 'elastic'). "
+                "Required for both local and Elastic Cloud setups unless API keys are used."
+            ),
+        ),
+        SecretStrInput(
+            name="password",
+            display_name="Password",
+            value="",
+            advanced=False,
+            info=(
+                "Elasticsearch password for the specified user. "
+                "Required for both local and Elastic Cloud setups unless API keys are used."
+            ),
+        ),
+        HandleInput(
+            name="embedding",
+            display_name="Embedding",
+            input_types=["Embeddings"],
+        ),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Type",
+            options=["similarity", "mmr"],
+            value="similarity",
+            advanced=True,
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=4,
+        ),
+        FloatInput(
+            name="search_score_threshold",
+            display_name="Search Score Threshold",
+            info="Minimum similarity score threshold for search results.",
+            value=0.0,
+            advanced=True,
+        ),
+        SecretStrInput(
+            name="api_key",
+            display_name="Elastic API Key",
+            value="",
+            advanced=True,
+            info="API Key for Elastic Cloud authentication. If used, 'username' and 'password' are not required.",
+        ),
+        BoolInput(
+            name="verify_certs",
+            display_name="Verify SSL Certificates",
+            value=True,
+            advanced=True,
+            info="Whether to verify SSL certificates when connecting to Elasticsearch.",
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> ElasticsearchStore:
+        """Builds the Elasticsearch Vector Store object."""
+        if self.cloud_id and self.elasticsearch_url:
+            msg = (
+                "Both 'cloud_id' and 'elasticsearch_url' provided. "
+                "Please use only one based on your deployment (Cloud or Local)."
+            )
+            raise ValueError(msg)
+
+        es_params = {
+            "index_name": self.index_name,
+            "embedding": self.embedding,
+            "es_user": self.username or None,
+            "es_password": self.password or None,
+        }
+
+        if self.cloud_id:
+            es_params["es_cloud_id"] = self.cloud_id
+        else:
+            es_params["es_url"] = self.elasticsearch_url
+
+        if self.api_key:
+            es_params["api_key"] = self.api_key
+
+        # Check if we need to verify SSL certificates
+        if self.verify_certs is False:
+            # Build client parameters for Elasticsearch constructor
+            client_params: dict[str, Any] = {}
+            client_params["verify_certs"] = False
+
+            if self.cloud_id:
+                client_params["cloud_id"] = self.cloud_id
+            else:
+                client_params["hosts"] = [self.elasticsearch_url]
+
+            if self.api_key:
+                client_params["api_key"] = self.api_key
+            elif self.username and self.password:
+                client_params["basic_auth"] = (self.username, self.password)
+
+            es_client = Elasticsearch(**client_params)
+            es_params["es_connection"] = es_client
+
+        elasticsearch = ElasticsearchStore(**es_params)
+
+        # If documents are provided, add them to the store
+        if self.ingest_data:
+            documents = self._prepare_documents()
+            if documents:
+                elasticsearch.add_documents(documents)
+
+        return elasticsearch
+
+    def _prepare_documents(self) -> list[Document]:
+        """Prepares documents from the input data to add to the vector store."""
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for data in self.ingest_data:
+            if isinstance(data, Data):
+                documents.append(data.to_lc_document())
+            else:
+                error_message = "Vector Store Inputs must be Data objects."
+                self.log(error_message)
+                raise TypeError(error_message)
+        return documents
+
+    def _add_documents_to_vector_store(self, vector_store: "ElasticsearchStore") -> None:
+        """Adds documents to the Vector Store."""
+        documents = self._prepare_documents()
+        if documents and self.embedding:
+            self.log(f"Adding {len(documents)} documents to the Vector Store.")
+            vector_store.add_documents(documents)
+        else:
+            self.log("No documents to add to the Vector Store.")
+
+    def search(self, query: str | None = None) -> list[dict[str, Any]]:
+        """Search for similar documents in the vector store or retrieve all documents if no query is provided."""
+        vector_store = self.build_vector_store()
+        search_kwargs = {
+            "k": self.number_of_results,
+            "score_threshold": self.search_score_threshold,
+        }
+
+        if query:
+            search_type = self.search_type.lower()
+            if search_type not in {"similarity", "mmr"}:
+                msg = f"Invalid search type: {self.search_type}"
+                self.log(msg)
+                raise ValueError(msg)
+            try:
+                if search_type == "similarity":
+                    results = vector_store.similarity_search_with_score(query, **search_kwargs)
+                elif search_type == "mmr":
+                    results = vector_store.max_marginal_relevance_search(query, **search_kwargs)
+            except Exception as e:
+                msg = (
+                    "Error occurred while querying the Elasticsearch VectorStore,"
+                    " there is no Data into the VectorStore."
+                )
+                self.log(msg)
+                raise ValueError(msg) from e
+            return [
+                {"page_content": doc.page_content, "metadata": doc.metadata, "score": score} for doc, score in results
+            ]
+        results = self.get_all_documents(vector_store, **search_kwargs)
+        return [{"page_content": doc.page_content, "metadata": doc.metadata, "score": score} for doc, score in results]
+
+    def get_all_documents(self, vector_store: ElasticsearchStore, **kwargs) -> list[tuple[Document, float]]:
+        """Retrieve all documents from the vector store."""
+        client = vector_store.client
+        index_name = self.index_name
+
+        query = {
+            "query": {"match_all": {}},
+            "size": kwargs.get("k", self.number_of_results),
+        }
+
+        response = client.search(index=index_name, body=query)
+
+        results = []
+        for hit in response["hits"]["hits"]:
+            doc = Document(
+                page_content=hit["_source"].get("text", ""),
+                metadata=hit["_source"].get("metadata", {}),
+            )
+            score = hit["_score"]
+            results.append((doc, score))
+
+        return results
+
+    def search_documents(self) -> list[Data]:
+        """Search for documents in the vector store based on the search input.
+
+        If no search input is provided, retrieve all documents.
+        """
+        results = self.search(self.search_query)
+        retrieved_data = [
+            Data(
+                text=result["page_content"],
+                file_path=result["metadata"].get("file_path", ""),
+            )
+            for result in results
+        ]
+        self.status = retrieved_data
+        return retrieved_data
+
+    def get_retriever_kwargs(self):
+        """Get the keyword arguments for the retriever."""
+        return {
+            "search_type": self.search_type.lower(),
+            "search_kwargs": {
+                "k": self.number_of_results,
+                "score_threshold": self.search_score_threshold,
+            },
+        }

lfx/components/vectorstores/faiss.py

@@ -0,0 +1,111 @@
+from pathlib import Path
+
+from langchain_community.vectorstores import FAISS
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import BoolInput, HandleInput, IntInput, StrInput
+from lfx.schema.data import Data
+
+
+class FaissVectorStoreComponent(LCVectorStoreComponent):
+    """FAISS Vector Store with search capabilities."""
+
+    display_name: str = "FAISS"
+    description: str = "FAISS Vector Store with search capabilities"
+    name = "FAISS"
+    icon = "FAISS"
+
+    inputs = [
+        StrInput(
+            name="index_name",
+            display_name="Index Name",
+            value="langflow_index",
+        ),
+        StrInput(
+            name="persist_directory",
+            display_name="Persist Directory",
+            info="Path to save the FAISS index. It will be relative to where Langflow is running.",
+        ),
+        *LCVectorStoreComponent.inputs,
+        BoolInput(
+            name="allow_dangerous_deserialization",
+            display_name="Allow Dangerous Deserialization",
+            info="Set to True to allow loading pickle files from untrusted sources. "
+            "Only enable this if you trust the source of the data.",
+            advanced=True,
+            value=True,
+        ),
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=4,
+        ),
+    ]
+
+    @staticmethod
+    def resolve_path(path: str) -> str:
+        """Resolve the path relative to the Langflow root.
+
+        Args:
+            path: The path to resolve
+        Returns:
+            str: The resolved path as a string
+        """
+        return str(Path(path).resolve())
+
+    def get_persist_directory(self) -> Path:
+        """Returns the resolved persist directory path or the current directory if not set."""
+        if self.persist_directory:
+            return Path(self.resolve_path(self.persist_directory))
+        return Path()
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> FAISS:
+        """Builds the FAISS object."""
+        path = self.get_persist_directory()
+        path.mkdir(parents=True, exist_ok=True)
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        faiss = FAISS.from_documents(documents=documents, embedding=self.embedding)
+        faiss.save_local(str(path), self.index_name)
+        return faiss
+
+    def search_documents(self) -> list[Data]:
+        """Search for documents in the FAISS vector store."""
+        path = self.get_persist_directory()
+        index_path = path / f"{self.index_name}.faiss"
+
+        if not index_path.exists():
+            vector_store = self.build_vector_store()
+        else:
+            vector_store = FAISS.load_local(
+                folder_path=str(path),
+                embeddings=self.embedding,
+                index_name=self.index_name,
+                allow_dangerous_deserialization=self.allow_dangerous_deserialization,
+            )
+
+        if not vector_store:
+            msg = "Failed to load the FAISS index."
+            raise ValueError(msg)
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+            return docs_to_data(docs)
+        return []

lfx/components/vectorstores/graph_rag.py

@@ -0,0 +1,141 @@
+import inspect
+from abc import ABC
+
+import graph_retriever.strategies as strategies_module
+from langchain_graph_retriever import GraphRetriever
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent
+from lfx.helpers.data import docs_to_data
+from lfx.inputs.inputs import DropdownInput, HandleInput, MultilineInput, NestedDictInput, StrInput
+from lfx.schema.data import Data
+
+
+def traversal_strategies() -> list[str]:
+    """Retrieves a list of class names from the strategies_module.
+
+    This function uses the `inspect` module to get all the class members
+    from the `strategies_module` and returns their names as a list of strings.
+
+    Returns:
+        list[str]: A list of strategy class names.
+    """
+    classes = inspect.getmembers(strategies_module, inspect.isclass)
+    return [name for name, cls in classes if ABC not in cls.__bases__]
+
+
+class GraphRAGComponent(LCVectorStoreComponent):
+    """GraphRAGComponent is a component for performing Graph RAG traversal in a vector store.
+
+    Attributes:
+        display_name (str): The display name of the component.
+        description (str): A brief description of the component.
+        name (str): The name of the component.
+        icon (str): The icon representing the component.
+        inputs (list): A list of input configurations for the component.
+
+    Methods:
+        _build_search_args():
+            Builds the arguments required for the search operation.
+        search_documents() -> list[Data]:
+            Searches for documents using the specified strategy, edge definition, and query.
+        _edge_definition_from_input() -> tuple:
+            Processes the edge definition input and returns it as a tuple.
+    """
+
+    display_name: str = "Graph RAG"
+    description: str = "Graph RAG traversal for vector store."
+    name = "Graph RAG"
+    icon: str = "AstraDB"
+
+    inputs = [
+        HandleInput(
+            name="embedding_model",
+            display_name="Embedding Model",
+            input_types=["Embeddings"],
+            info="Specify the Embedding Model. Not required for Astra Vectorize collections.",
+            required=False,
+        ),
+        HandleInput(
+            name="vector_store",
+            display_name="Vector Store Connection",
+            input_types=["VectorStore"],
+            info="Connection to Vector Store.",
+        ),
+        StrInput(
+            name="edge_definition",
+            display_name="Edge Definition",
+            info="Edge definition for the graph traversal.",
+        ),
+        DropdownInput(
+            name="strategy",
+            display_name="Traversal Strategies",
+            options=traversal_strategies(),
+        ),
+        MultilineInput(
+            name="search_query",
+            display_name="Search Query",
+            tool_mode=True,
+        ),
+        NestedDictInput(
+            name="graphrag_strategy_kwargs",
+            display_name="Strategy Parameters",
+            info=(
+                "Optional dictionary of additional parameters for the retrieval strategy. "
+                "Please see https://datastax.github.io/graph-rag/reference/graph_retriever/strategies/ for details."
+            ),
+            advanced=True,
+        ),
+    ]
+
+    def search_documents(self) -> list[Data]:
+        """Searches for documents using the graph retriever based on the selected strategy, edge definition, and query.
+
+        Returns:
+            list[Data]: A list of retrieved documents.
+
+        Raises:
+            AttributeError: If there is an issue with attribute access.
+            TypeError: If there is a type mismatch.
+            ValueError: If there is a value error.
+        """
+        additional_params = self.graphrag_strategy_kwargs or {}
+
+        # Invoke the graph retriever based on the selected strategy, edge definition, and query
+        strategy_class = getattr(strategies_module, self.strategy)
+        retriever = GraphRetriever(
+            store=self.vector_store,
+            edges=[self._evaluate_edge_definition_input()],
+            strategy=strategy_class(**additional_params),
+        )
+
+        return docs_to_data(retriever.invoke(self.search_query))
+
+    def _edge_definition_from_input(self) -> tuple:
+        """Generates the edge definition from the input data.
+
+        Returns:
+            tuple: A tuple representing the edge definition.
+        """
+        values = self.edge_definition.split(",")
+        values = [value.strip() for value in values]
+
+        return tuple(values)
+
+    def _evaluate_edge_definition_input(self) -> tuple:
+        from graph_retriever.edges.metadata import Id
+
+        """Evaluates the edge definition, converting any function calls from strings.
+
+        Args:
+            edge_definition (tuple): The edge definition to evaluate.
+
+        Returns:
+            tuple: The evaluated edge definition.
+        """
+        evaluated_values = []
+        for value in self._edge_definition_from_input():
+            if value == "Id()":
+                evaluated_values.append(Id())  # Evaluate Id() as a function call
+            else:
+                evaluated_values.append(value)
+        return tuple(evaluated_values)