lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/vectorstores/hcd.py

@@ -0,0 +1,314 @@
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.inputs.inputs import DictInput, FloatInput
+from lfx.io import (
+    BoolInput,
+    DropdownInput,
+    HandleInput,
+    IntInput,
+    MultilineInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class HCDVectorStoreComponent(LCVectorStoreComponent):
+    display_name: str = "Hyper-Converged Database"
+    description: str = "Implementation of Vector Store using Hyper-Converged Database (HCD) with search capabilities"
+    name = "HCD"
+    icon: str = "HCD"
+
+    inputs = [
+        StrInput(
+            name="collection_name",
+            display_name="Collection Name",
+            info="The name of the collection within HCD where the vectors will be stored.",
+            required=True,
+        ),
+        StrInput(
+            name="username",
+            display_name="HCD Username",
+            info="Authentication username for accessing HCD.",
+            value="hcd-superuser",
+            required=True,
+        ),
+        SecretStrInput(
+            name="password",
+            display_name="HCD Password",
+            info="Authentication password for accessing HCD.",
+            value="HCD_PASSWORD",
+            required=True,
+        ),
+        SecretStrInput(
+            name="api_endpoint",
+            display_name="HCD API Endpoint",
+            info="API endpoint URL for the HCD service.",
+            value="HCD_API_ENDPOINT",
+            required=True,
+        ),
+        *LCVectorStoreComponent.inputs,
+        StrInput(
+            name="namespace",
+            display_name="Namespace",
+            info="Optional namespace within HCD to use for the collection.",
+            value="default_namespace",
+            advanced=True,
+        ),
+        MultilineInput(
+            name="ca_certificate",
+            display_name="CA Certificate",
+            info="Optional CA certificate for TLS connections to HCD.",
+            advanced=True,
+        ),
+        DropdownInput(
+            name="metric",
+            display_name="Metric",
+            info="Optional distance metric for vector comparisons in the vector store.",
+            options=["cosine", "dot_product", "euclidean"],
+            advanced=True,
+        ),
+        IntInput(
+            name="batch_size",
+            display_name="Batch Size",
+            info="Optional number of data to process in a single batch.",
+            advanced=True,
+        ),
+        IntInput(
+            name="bulk_insert_batch_concurrency",
+            display_name="Bulk Insert Batch Concurrency",
+            info="Optional concurrency level for bulk insert operations.",
+            advanced=True,
+        ),
+        IntInput(
+            name="bulk_insert_overwrite_concurrency",
+            display_name="Bulk Insert Overwrite Concurrency",
+            info="Optional concurrency level for bulk insert operations that overwrite existing data.",
+            advanced=True,
+        ),
+        IntInput(
+            name="bulk_delete_concurrency",
+            display_name="Bulk Delete Concurrency",
+            info="Optional concurrency level for bulk delete operations.",
+            advanced=True,
+        ),
+        DropdownInput(
+            name="setup_mode",
+            display_name="Setup Mode",
+            info="Configuration mode for setting up the vector store, with options like 'Sync', 'Async', or 'Off'.",
+            options=["Sync", "Async", "Off"],
+            advanced=True,
+            value="Sync",
+        ),
+        BoolInput(
+            name="pre_delete_collection",
+            display_name="Pre Delete Collection",
+            info="Boolean flag to determine whether to delete the collection before creating a new one.",
+            advanced=True,
+        ),
+        StrInput(
+            name="metadata_indexing_include",
+            display_name="Metadata Indexing Include",
+            info="Optional list of metadata fields to include in the indexing.",
+            advanced=True,
+        ),
+        HandleInput(
+            name="embedding",
+            display_name="Embedding or Astra Vectorize",
+            input_types=["Embeddings", "dict"],
+            # TODO: This should be optional, but need to refactor langchain-astradb first.
+            info="Allows either an embedding model or an Astra Vectorize configuration.",
+        ),
+        StrInput(
+            name="metadata_indexing_exclude",
+            display_name="Metadata Indexing Exclude",
+            info="Optional list of metadata fields to exclude from the indexing.",
+            advanced=True,
+        ),
+        StrInput(
+            name="collection_indexing_policy",
+            display_name="Collection Indexing Policy",
+            info="Optional dictionary defining the indexing policy for the collection.",
+            advanced=True,
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=4,
+        ),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Type",
+            info="Search type to use",
+            options=["Similarity", "Similarity with score threshold", "MMR (Max Marginal Relevance)"],
+            value="Similarity",
+            advanced=True,
+        ),
+        FloatInput(
+            name="search_score_threshold",
+            display_name="Search Score Threshold",
+            info="Minimum similarity score threshold for search results. "
+            "(when using 'Similarity with score threshold')",
+            value=0,
+            advanced=True,
+        ),
+        DictInput(
+            name="search_filter",
+            display_name="Search Metadata Filter",
+            info="Optional dictionary of filters to apply to the search query.",
+            advanced=True,
+            is_list=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self):
+        try:
+            from langchain_astradb import AstraDBVectorStore
+            from langchain_astradb.utils.astradb import SetupMode
+        except ImportError as e:
+            msg = (
+                "Could not import langchain Astra DB integration package. "
+                "Please install it with `pip install langchain-astradb`."
+            )
+            raise ImportError(msg) from e
+
+        try:
+            from astrapy.authentication import UsernamePasswordTokenProvider
+            from astrapy.constants import Environment
+        except ImportError as e:
+            msg = "Could not import astrapy integration package. Please install it with `pip install astrapy`."
+            raise ImportError(msg) from e
+
+        try:
+            if not self.setup_mode:
+                self.setup_mode = self._inputs["setup_mode"].options[0]
+
+            setup_mode_value = SetupMode[self.setup_mode.upper()]
+        except KeyError as e:
+            msg = f"Invalid setup mode: {self.setup_mode}"
+            raise ValueError(msg) from e
+
+        if not isinstance(self.embedding, dict):
+            embedding_dict = {"embedding": self.embedding}
+        else:
+            from astrapy.info import VectorServiceOptions
+
+            dict_options = self.embedding.get("collection_vector_service_options", {})
+            dict_options["authentication"] = {
+                k: v for k, v in dict_options.get("authentication", {}).items() if k and v
+            }
+            dict_options["parameters"] = {k: v for k, v in dict_options.get("parameters", {}).items() if k and v}
+            embedding_dict = {"collection_vector_service_options": VectorServiceOptions.from_dict(dict_options)}
+            collection_embedding_api_key = self.embedding.get("collection_embedding_api_key")
+            if collection_embedding_api_key:
+                embedding_dict["collection_embedding_api_key"] = collection_embedding_api_key
+
+        token_provider = UsernamePasswordTokenProvider(self.username, self.password)
+        vector_store_kwargs = {
+            **embedding_dict,
+            "collection_name": self.collection_name,
+            "token": token_provider,
+            "api_endpoint": self.api_endpoint,
+            "namespace": self.namespace,
+            "metric": self.metric or None,
+            "batch_size": self.batch_size or None,
+            "bulk_insert_batch_concurrency": self.bulk_insert_batch_concurrency or None,
+            "bulk_insert_overwrite_concurrency": self.bulk_insert_overwrite_concurrency or None,
+            "bulk_delete_concurrency": self.bulk_delete_concurrency or None,
+            "setup_mode": setup_mode_value,
+            "pre_delete_collection": self.pre_delete_collection or False,
+            "environment": Environment.HCD,
+        }
+
+        if self.metadata_indexing_include:
+            vector_store_kwargs["metadata_indexing_include"] = self.metadata_indexing_include
+        elif self.metadata_indexing_exclude:
+            vector_store_kwargs["metadata_indexing_exclude"] = self.metadata_indexing_exclude
+        elif self.collection_indexing_policy:
+            vector_store_kwargs["collection_indexing_policy"] = self.collection_indexing_policy
+
+        try:
+            vector_store = AstraDBVectorStore(**vector_store_kwargs)
+        except Exception as e:
+            msg = f"Error initializing AstraDBVectorStore: {e}"
+            raise ValueError(msg) from e
+
+        self._add_documents_to_vector_store(vector_store)
+        return vector_store
+
+    def _add_documents_to_vector_store(self, vector_store) -> None:
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                msg = "Vector Store Inputs must be Data objects."
+                raise TypeError(msg)
+
+        if documents:
+            self.log(f"Adding {len(documents)} documents to the Vector Store.")
+            try:
+                vector_store.add_documents(documents)
+            except Exception as e:
+                msg = f"Error adding documents to AstraDBVectorStore: {e}"
+                raise ValueError(msg) from e
+        else:
+            self.log("No documents to add to the Vector Store.")
+
+    def _map_search_type(self) -> str:
+        if self.search_type == "Similarity with score threshold":
+            return "similarity_score_threshold"
+        if self.search_type == "MMR (Max Marginal Relevance)":
+            return "mmr"
+        return "similarity"
+
+    def _build_search_args(self):
+        args = {
+            "k": self.number_of_results,
+            "score_threshold": self.search_score_threshold,
+        }
+
+        if self.search_filter:
+            clean_filter = {k: v for k, v in self.search_filter.items() if k and v}
+            if len(clean_filter) > 0:
+                args["filter"] = clean_filter
+        return args
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        self.log(f"Search query: {self.search_query}")
+        self.log(f"Search type: {self.search_type}")
+        self.log(f"Number of results: {self.number_of_results}")
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            try:
+                search_type = self._map_search_type()
+                search_args = self._build_search_args()
+
+                docs = vector_store.search(query=self.search_query, search_type=search_type, **search_args)
+            except Exception as e:
+                msg = f"Error performing search in AstraDBVectorStore: {e}"
+                raise ValueError(msg) from e
+
+            self.log(f"Retrieved documents: {len(docs)}")
+
+            data = docs_to_data(docs)
+            self.log(f"Converted documents to data: {len(data)}")
+            self.status = data
+            return data
+        self.log("No search input provided. Skipping search.")
+        return []
+
+    def get_retriever_kwargs(self):
+        search_args = self._build_search_args()
+        return {
+            "search_type": self._map_search_type(),
+            "search_kwargs": search_args,
+        }

lfx/components/vectorstores/local_db.py

@@ -0,0 +1,261 @@
+from copy import deepcopy
+from pathlib import Path
+
+from langchain_chroma import Chroma
+from typing_extensions import override
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.base.vectorstores.utils import chroma_collection_to_data
+from lfx.inputs.inputs import MultilineInput
+from lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, MessageTextInput, TabInput
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.template.field.base import Output
+
+
+class LocalDBComponent(LCVectorStoreComponent):
+    """Chroma Vector Store with search capabilities."""
+
+    display_name: str = "Local DB"
+    description: str = "Local Vector Store with search capabilities"
+    name = "LocalDB"
+    icon = "database"
+    legacy = True
+
+    inputs = [
+        TabInput(
+            name="mode",
+            display_name="Mode",
+            options=["Ingest", "Retrieve"],
+            info="Select the operation mode",
+            value="Ingest",
+            real_time_refresh=True,
+            show=True,
+        ),
+        MessageTextInput(
+            name="collection_name",
+            display_name="Collection Name",
+            value="langflow",
+            required=True,
+        ),
+        MessageTextInput(
+            name="persist_directory",
+            display_name="Persist Directory",
+            info=(
+                "Custom base directory to save the vector store. "
+                "Collections will be stored under '{directory}/vector_stores/{collection_name}'. "
+                "If not specified, it will use your system's cache folder."
+            ),
+            advanced=True,
+        ),
+        DropdownInput(
+            name="existing_collections",
+            display_name="Existing Collections",
+            options=[],  # Will be populated dynamically
+            info="Select a previously created collection to search through its stored data.",
+            show=False,
+            combobox=True,
+        ),
+        HandleInput(name="embedding", display_name="Embedding", required=True, input_types=["Embeddings"]),
+        BoolInput(
+            name="allow_duplicates",
+            display_name="Allow Duplicates",
+            advanced=True,
+            info="If false, will not add documents that are already in the Vector Store.",
+        ),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Type",
+            options=["Similarity", "MMR"],
+            value="Similarity",
+            advanced=True,
+        ),
+        HandleInput(
+            name="ingest_data",
+            display_name="Ingest Data",
+            input_types=["Data", "DataFrame"],
+            is_list=True,
+            info="Data to store. It will be embedded and indexed for semantic search.",
+            show=True,
+        ),
+        MultilineInput(
+            name="search_query",
+            display_name="Search Query",
+            tool_mode=True,
+            info="Enter text to search for similar content in the selected collection.",
+            show=False,
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=10,
+        ),
+        IntInput(
+            name="limit",
+            display_name="Limit",
+            advanced=True,
+            info="Limit the number of records to compare when Allow Duplicates is False.",
+        ),
+    ]
+    outputs = [
+        Output(display_name="DataFrame", name="dataframe", method="perform_search"),
+    ]
+
+    def get_vector_store_directory(self, base_dir: str | Path) -> Path:
+        """Get the full directory path for a collection."""
+        # Ensure base_dir is a Path object
+        base_dir = Path(base_dir)
+        # Create the full path: base_dir/vector_stores/collection_name
+        full_path = base_dir / "vector_stores" / self.collection_name
+        # Create the directory if it doesn't exist
+        full_path.mkdir(parents=True, exist_ok=True)
+        return full_path
+
+    def get_default_persist_dir(self) -> str:
+        """Get the default persist directory from cache."""
+        from lfx.services.cache.utils import CACHE_DIR
+
+        return str(self.get_vector_store_directory(CACHE_DIR))
+
+    def list_existing_collections(self) -> list[str]:
+        """List existing vector store collections from the persist directory."""
+        from lfx.services.cache.utils import CACHE_DIR
+
+        # Get the base directory (either custom or cache)
+        base_dir = Path(self.persist_directory) if self.persist_directory else Path(CACHE_DIR)
+        # Get the vector_stores subdirectory
+        vector_stores_dir = base_dir / "vector_stores"
+        if not vector_stores_dir.exists():
+            return []
+
+        return [d.name for d in vector_stores_dir.iterdir() if d.is_dir()]
+
+    def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:
+        """Update the build configuration when the mode changes."""
+        if field_name == "mode":
+            # Hide all dynamic fields by default
+            dynamic_fields = [
+                "ingest_data",
+                "search_query",
+                "search_type",
+                "number_of_results",
+                "existing_collections",
+                "collection_name",
+                "embedding",
+                "allow_duplicates",
+                "limit",
+            ]
+            for field in dynamic_fields:
+                if field in build_config:
+                    build_config[field]["show"] = False
+
+            # Show/hide fields based on selected mode
+            if field_value == "Ingest":
+                if "ingest_data" in build_config:
+                    build_config["ingest_data"]["show"] = True
+                if "collection_name" in build_config:
+                    build_config["collection_name"]["show"] = True
+                    build_config["collection_name"]["display_name"] = "Name Your Collection"
+                if "persist" in build_config:
+                    build_config["persist"]["show"] = True
+                if "persist_directory" in build_config:
+                    build_config["persist_directory"]["show"] = True
+                if "embedding" in build_config:
+                    build_config["embedding"]["show"] = True
+                if "allow_duplicates" in build_config:
+                    build_config["allow_duplicates"]["show"] = True
+                if "limit" in build_config:
+                    build_config["limit"]["show"] = True
+            elif field_value == "Retrieve":
+                if "persist" in build_config:
+                    build_config["persist"]["show"] = False
+                build_config["search_query"]["show"] = True
+                build_config["search_type"]["show"] = True
+                build_config["number_of_results"]["show"] = True
+                build_config["embedding"]["show"] = True
+                build_config["collection_name"]["show"] = False
+                # Show existing collections dropdown and update its options
+                if "existing_collections" in build_config:
+                    build_config["existing_collections"]["show"] = True
+                    build_config["existing_collections"]["options"] = self.list_existing_collections()
+                # Hide collection_name in Retrieve mode since we use existing_collections
+        elif field_name == "existing_collections":
+            # Update collection_name when an existing collection is selected
+            if "collection_name" in build_config:
+                build_config["collection_name"]["value"] = field_value
+
+        return build_config
+
+    @override
+    @check_cached_vector_store
+    def build_vector_store(self) -> Chroma:
+        """Builds the Chroma object."""
+        try:
+            from langchain_chroma import Chroma
+        except ImportError as e:
+            msg = "Could not import Chroma integration package. Please install it with `pip install langchain-chroma`."
+            raise ImportError(msg) from e
+        # Chroma settings
+        # chroma_settings = None
+        if self.existing_collections:
+            self.collection_name = self.existing_collections
+
+        # Use user-provided directory or default cache directory
+        if self.persist_directory:
+            base_dir = self.resolve_path(self.persist_directory)
+            persist_directory = str(self.get_vector_store_directory(base_dir))
+            logger.debug(f"Using custom persist directory: {persist_directory}")
+        else:
+            persist_directory = self.get_default_persist_dir()
+            logger.debug(f"Using default persist directory: {persist_directory}")
+
+        chroma = Chroma(
+            persist_directory=persist_directory,
+            client=None,
+            embedding_function=self.embedding,
+            collection_name=self.collection_name,
+        )
+
+        self._add_documents_to_vector_store(chroma)
+        self.status = chroma_collection_to_data(chroma.get(limit=self.limit))
+        return chroma
+
+    def _add_documents_to_vector_store(self, vector_store: "Chroma") -> None:
+        """Adds documents to the Vector Store."""
+        ingest_data: list | Data | DataFrame = self.ingest_data
+        if not ingest_data:
+            self.status = ""
+            return
+
+        # Convert DataFrame to Data if needed using parent's method
+        ingest_data = self._prepare_ingest_data()
+
+        stored_documents_without_id = []
+        if self.allow_duplicates:
+            stored_data = []
+        else:
+            stored_data = chroma_collection_to_data(vector_store.get(limit=self.limit))
+            for value in deepcopy(stored_data):
+                del value.id
+                stored_documents_without_id.append(value)
+
+        documents = []
+        for _input in ingest_data or []:
+            if isinstance(_input, Data):
+                if _input not in stored_documents_without_id:
+                    documents.append(_input.to_lc_document())
+            else:
+                msg = "Vector Store Inputs must be Data objects."
+                raise TypeError(msg)
+
+        if documents and self.embedding is not None:
+            self.log(f"Adding {len(documents)} documents to the Vector Store.")
+            vector_store.add_documents(documents)
+        else:
+            self.log("No documents to add to the Vector Store.")
+
+    def perform_search(self) -> DataFrame:
+        return DataFrame(self.search_documents())

lfx/components/vectorstores/milvus.py

@@ -0,0 +1,115 @@
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import (
+    BoolInput,
+    DictInput,
+    DropdownInput,
+    FloatInput,
+    HandleInput,
+    IntInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class MilvusVectorStoreComponent(LCVectorStoreComponent):
+    """Milvus vector store with search capabilities."""
+
+    display_name: str = "Milvus"
+    description: str = "Milvus vector store with search capabilities"
+    name = "Milvus"
+    icon = "Milvus"
+
+    inputs = [
+        StrInput(name="collection_name", display_name="Collection Name", value="langflow"),
+        StrInput(name="collection_description", display_name="Collection Description", value=""),
+        StrInput(
+            name="uri",
+            display_name="Connection URI",
+            value="http://localhost:19530",
+        ),
+        SecretStrInput(
+            name="password",
+            display_name="Token",
+            value="",
+            info="Ignore this field if no token is required to make connection.",
+        ),
+        DictInput(name="connection_args", display_name="Other Connection Arguments", advanced=True),
+        StrInput(name="primary_field", display_name="Primary Field Name", value="pk"),
+        StrInput(name="text_field", display_name="Text Field Name", value="text"),
+        StrInput(name="vector_field", display_name="Vector Field Name", value="vector"),
+        DropdownInput(
+            name="consistency_level",
+            display_name="Consistencey Level",
+            options=["Bounded", "Session", "Strong", "Eventual"],
+            value="Session",
+            advanced=True,
+        ),
+        DictInput(name="index_params", display_name="Index Parameters", advanced=True),
+        DictInput(name="search_params", display_name="Search Parameters", advanced=True),
+        BoolInput(name="drop_old", display_name="Drop Old Collection", value=False, advanced=True),
+        FloatInput(name="timeout", display_name="Timeout", advanced=True),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self):
+        try:
+            from langchain_milvus.vectorstores import Milvus as LangchainMilvus
+        except ImportError as e:
+            msg = "Could not import Milvus integration package. Please install it with `pip install langchain-milvus`."
+            raise ImportError(msg) from e
+        self.connection_args.update(uri=self.uri, token=self.password)
+        milvus_store = LangchainMilvus(
+            embedding_function=self.embedding,
+            collection_name=self.collection_name,
+            collection_description=self.collection_description,
+            connection_args=self.connection_args,
+            consistency_level=self.consistency_level,
+            index_params=self.index_params,
+            search_params=self.search_params,
+            drop_old=self.drop_old,
+            auto_id=True,
+            primary_field=self.primary_field,
+            text_field=self.text_field,
+            vector_field=self.vector_field,
+            timeout=self.timeout,
+        )
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents:
+            milvus_store.add_documents(documents)
+
+        return milvus_store
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []