lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/vectara/vectara.py

@@ -0,0 +1,97 @@
+from typing import TYPE_CHECKING
+
+from langchain_community.vectorstores import Vectara
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+if TYPE_CHECKING:
+    from lfx.schema.dataframe import DataFrame
+
+
+class VectaraVectorStoreComponent(LCVectorStoreComponent):
+    """Vectara Vector Store with search capabilities."""
+
+    display_name: str = "Vectara"
+    description: str = "Vectara Vector Store with search capabilities"
+    name = "Vectara"
+    icon = "Vectara"
+
+    inputs = [
+        StrInput(name="vectara_customer_id", display_name="Vectara Customer ID", required=True),
+        StrInput(name="vectara_corpus_id", display_name="Vectara Corpus ID", required=True),
+        SecretStrInput(name="vectara_api_key", display_name="Vectara API Key", required=True),
+        HandleInput(
+            name="embedding",
+            display_name="Embedding",
+            input_types=["Embeddings"],
+        ),
+        *LCVectorStoreComponent.inputs,
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> Vectara:
+        """Builds the Vectara object."""
+        try:
+            from langchain_community.vectorstores import Vectara
+        except ImportError as e:
+            msg = "Could not import Vectara. Please install it with `pip install langchain-community`."
+            raise ImportError(msg) from e
+
+        vectara = Vectara(
+            vectara_customer_id=self.vectara_customer_id,
+            vectara_corpus_id=self.vectara_corpus_id,
+            vectara_api_key=self.vectara_api_key,
+        )
+
+        self._add_documents_to_vector_store(vectara)
+        return vectara
+
+    def _add_documents_to_vector_store(self, vector_store: Vectara) -> None:
+        """Adds documents to the Vector Store."""
+        ingest_data: list | Data | DataFrame = self.ingest_data
+        if not ingest_data:
+            self.status = "No documents to add to Vectara"
+            return
+
+        # Convert DataFrame to Data if needed using parent's method
+        ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents:
+            self.log(f"Adding {len(documents)} documents to Vectara.")
+            vector_store.add_documents(documents)
+            self.status = f"Added {len(documents)} documents to Vectara"
+        else:
+            self.log("No documents to add to Vectara.")
+            self.status = "No valid documents to add to Vectara"
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+
+            data = docs_to_data(docs)
+            self.status = f"Found {len(data)} results for the query: {self.search_query}"
+            return data
+        self.status = "No search query provided"
+        return []

lfx/components/vectara/vectara_rag.py

@@ -0,0 +1,164 @@
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing.range_spec import RangeSpec
+from lfx.io import DropdownInput, FloatInput, IntInput, MessageTextInput, Output, SecretStrInput, StrInput
+from lfx.schema.message import Message
+
+
+class VectaraRagComponent(Component):
+    display_name = "Vectara RAG"
+    description = "Vectara's full end to end RAG"
+    documentation = "https://docs.vectara.com/docs"
+    icon = "Vectara"
+    name = "VectaraRAG"
+    SUMMARIZER_PROMPTS = [
+        "vectara-summary-ext-24-05-sml",
+        "vectara-summary-ext-24-05-med-omni",
+        "vectara-summary-ext-24-05-large",
+        "vectara-summary-ext-24-05-med",
+        "vectara-summary-ext-v1.3.0",
+    ]
+
+    RERANKER_TYPES = ["mmr", "rerank_multilingual_v1", "none"]
+
+    RESPONSE_LANGUAGES = [
+        "auto",
+        "eng",
+        "spa",
+        "fra",
+        "zho",
+        "deu",
+        "hin",
+        "ara",
+        "por",
+        "ita",
+        "jpn",
+        "kor",
+        "rus",
+        "tur",
+        "fas",
+        "vie",
+        "tha",
+        "heb",
+        "nld",
+        "ind",
+        "pol",
+        "ukr",
+        "ron",
+        "swe",
+        "ces",
+        "ell",
+        "ben",
+        "msa",
+        "urd",
+    ]
+
+    field_order = ["vectara_customer_id", "vectara_corpus_id", "vectara_api_key", "search_query", "reranker"]
+
+    inputs = [
+        StrInput(name="vectara_customer_id", display_name="Vectara Customer ID", required=True),
+        StrInput(name="vectara_corpus_id", display_name="Vectara Corpus ID", required=True),
+        SecretStrInput(name="vectara_api_key", display_name="Vectara API Key", required=True),
+        MessageTextInput(
+            name="search_query",
+            display_name="Search Query",
+            info="The query to receive an answer on.",
+            tool_mode=True,
+        ),
+        FloatInput(
+            name="lexical_interpolation",
+            display_name="Hybrid Search Factor",
+            range_spec=RangeSpec(min=0.005, max=0.1, step=0.005),
+            value=0.005,
+            advanced=True,
+            info="How much to weigh lexical scores compared to the embedding score. "
+            "0 means lexical search is not used at all, and 1 means only lexical search is used.",
+        ),
+        MessageTextInput(
+            name="filter",
+            display_name="Metadata Filters",
+            value="",
+            advanced=True,
+            info="The filter string to narrow the search to according to metadata attributes.",
+        ),
+        DropdownInput(
+            name="reranker",
+            display_name="Reranker Type",
+            options=RERANKER_TYPES,
+            value=RERANKER_TYPES[0],
+            info="How to rerank the retrieved search results.",
+        ),
+        IntInput(
+            name="reranker_k",
+            display_name="Number of Results to Rerank",
+            value=50,
+            range_spec=RangeSpec(min=1, max=100, step=1),
+            advanced=True,
+        ),
+        FloatInput(
+            name="diversity_bias",
+            display_name="Diversity Bias",
+            value=0.2,
+            range_spec=RangeSpec(min=0, max=1, step=0.01),
+            advanced=True,
+            info="Ranges from 0 to 1, with higher values indicating greater diversity (only applies to MMR reranker).",
+        ),
+        IntInput(
+            name="max_results",
+            display_name="Max Results to Summarize",
+            value=7,
+            range_spec=RangeSpec(min=1, max=100, step=1),
+            advanced=True,
+            info="The maximum number of search results to be available to the prompt.",
+        ),
+        DropdownInput(
+            name="response_lang",
+            display_name="Response Language",
+            options=RESPONSE_LANGUAGES,
+            value="eng",
+            advanced=True,
+            info="Use the ISO 639-1 or 639-3 language code or auto to automatically detect the language.",
+        ),
+        DropdownInput(
+            name="prompt",
+            display_name="Prompt Name",
+            options=SUMMARIZER_PROMPTS,
+            value=SUMMARIZER_PROMPTS[0],
+            advanced=True,
+            info="Only vectara-summary-ext-24-05-sml is for Growth customers; "
+            "all other prompts are for Scale customers only.",
+        ),
+    ]
+
+    outputs = [
+        Output(name="answer", display_name="Answer", method="generate_response"),
+    ]
+
+    def generate_response(
+        self,
+    ) -> Message:
+        text_output = ""
+
+        try:
+            from langchain_community.vectorstores import Vectara
+            from langchain_community.vectorstores.vectara import RerankConfig, SummaryConfig, VectaraQueryConfig
+        except ImportError as e:
+            msg = "Could not import Vectara. Please install it with `pip install langchain-community`."
+            raise ImportError(msg) from e
+
+        vectara = Vectara(self.vectara_customer_id, self.vectara_corpus_id, self.vectara_api_key)
+        rerank_config = RerankConfig(self.reranker, self.reranker_k, self.diversity_bias)
+        summary_config = SummaryConfig(
+            is_enabled=True, max_results=self.max_results, response_lang=self.response_lang, prompt_name=self.prompt
+        )
+        config = VectaraQueryConfig(
+            lambda_val=self.lexical_interpolation,
+            filter=self.filter,
+            summary_config=summary_config,
+            rerank_config=rerank_config,
+        )
+        rag = vectara.as_rag(config)
+        response = rag.invoke(self.search_query, config={"callbacks": self.get_langchain_callbacks()})
+
+        text_output = response["answer"]
+
+        return Message(text=text_output)

lfx/components/vectorstores/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .local_db import LocalDBComponent
+
+_dynamic_imports = {
+    "LocalDBComponent": "local_db",
+}
+
+__all__ = [
+    "LocalDBComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import vectorstore components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/vectorstores/local_db.py

@@ -0,0 +1,270 @@
+from copy import deepcopy
+from pathlib import Path
+
+from langchain_chroma import Chroma
+from typing_extensions import override
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.base.vectorstores.utils import chroma_collection_to_data
+from lfx.inputs.inputs import MultilineInput
+from lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, MessageTextInput, TabInput
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.template.field.base import Output
+from lfx.utils.validate_cloud import raise_error_if_astra_cloud_disable_component
+
+disable_component_in_astra_cloud_msg = (
+    "Local vector stores are not supported in S3/cloud mode. "
+    "Local vector stores require local file system access for persistence. "
+    "Please use cloud-based vector stores (Pinecone, Weaviate, etc.) or local storage mode."
+)
+
+
+class LocalDBComponent(LCVectorStoreComponent):
+    """Chroma Vector Store with search capabilities."""
+
+    display_name: str = "Local DB"
+    description: str = "Local Vector Store with search capabilities"
+    name = "LocalDB"
+    icon = "database"
+    legacy = True
+
+    inputs = [
+        TabInput(
+            name="mode",
+            display_name="Mode",
+            options=["Ingest", "Retrieve"],
+            info="Select the operation mode",
+            value="Ingest",
+            real_time_refresh=True,
+            show=True,
+        ),
+        MessageTextInput(
+            name="collection_name",
+            display_name="Collection Name",
+            value="langflow",
+            required=True,
+        ),
+        MessageTextInput(
+            name="persist_directory",
+            display_name="Persist Directory",
+            info=(
+                "Custom base directory to save the vector store. "
+                "Collections will be stored under '{directory}/vector_stores/{collection_name}'. "
+                "If not specified, it will use your system's cache folder."
+            ),
+            advanced=True,
+        ),
+        DropdownInput(
+            name="existing_collections",
+            display_name="Existing Collections",
+            options=[],  # Will be populated dynamically
+            info="Select a previously created collection to search through its stored data.",
+            show=False,
+            combobox=True,
+        ),
+        HandleInput(name="embedding", display_name="Embedding", required=True, input_types=["Embeddings"]),
+        BoolInput(
+            name="allow_duplicates",
+            display_name="Allow Duplicates",
+            advanced=True,
+            info="If false, will not add documents that are already in the Vector Store.",
+        ),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Type",
+            options=["Similarity", "MMR"],
+            value="Similarity",
+            advanced=True,
+        ),
+        HandleInput(
+            name="ingest_data",
+            display_name="Ingest Data",
+            input_types=["Data", "DataFrame"],
+            is_list=True,
+            info="Data to store. It will be embedded and indexed for semantic search.",
+            show=True,
+        ),
+        MultilineInput(
+            name="search_query",
+            display_name="Search Query",
+            tool_mode=True,
+            info="Enter text to search for similar content in the selected collection.",
+            show=False,
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=10,
+        ),
+        IntInput(
+            name="limit",
+            display_name="Limit",
+            advanced=True,
+            info="Limit the number of records to compare when Allow Duplicates is False.",
+        ),
+    ]
+    outputs = [
+        Output(display_name="DataFrame", name="dataframe", method="perform_search"),
+    ]
+
+    def get_vector_store_directory(self, base_dir: str | Path) -> Path:
+        """Get the full directory path for a collection."""
+        # Ensure base_dir is a Path object
+        base_dir = Path(base_dir)
+        # Create the full path: base_dir/vector_stores/collection_name
+        full_path = base_dir / "vector_stores" / self.collection_name
+        # Create the directory if it doesn't exist
+        full_path.mkdir(parents=True, exist_ok=True)
+        return full_path
+
+    def get_default_persist_dir(self) -> str:
+        """Get the default persist directory from cache."""
+        from lfx.services.cache.utils import CACHE_DIR
+
+        return str(self.get_vector_store_directory(CACHE_DIR))
+
+    def list_existing_collections(self) -> list[str]:
+        """List existing vector store collections from the persist directory."""
+        from lfx.services.cache.utils import CACHE_DIR
+
+        # Get the base directory (either custom or cache)
+        base_dir = Path(self.persist_directory) if self.persist_directory else Path(CACHE_DIR)
+        # Get the vector_stores subdirectory
+        vector_stores_dir = base_dir / "vector_stores"
+        if not vector_stores_dir.exists():
+            return []
+
+        return [d.name for d in vector_stores_dir.iterdir() if d.is_dir()]
+
+    def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:
+        """Update the build configuration when the mode changes."""
+        if field_name == "mode":
+            # Hide all dynamic fields by default
+            dynamic_fields = [
+                "ingest_data",
+                "search_query",
+                "search_type",
+                "number_of_results",
+                "existing_collections",
+                "collection_name",
+                "embedding",
+                "allow_duplicates",
+                "limit",
+            ]
+            for field in dynamic_fields:
+                if field in build_config:
+                    build_config[field]["show"] = False
+
+            # Show/hide fields based on selected mode
+            if field_value == "Ingest":
+                if "ingest_data" in build_config:
+                    build_config["ingest_data"]["show"] = True
+                if "collection_name" in build_config:
+                    build_config["collection_name"]["show"] = True
+                    build_config["collection_name"]["display_name"] = "Name Your Collection"
+                if "persist" in build_config:
+                    build_config["persist"]["show"] = True
+                if "persist_directory" in build_config:
+                    build_config["persist_directory"]["show"] = True
+                if "embedding" in build_config:
+                    build_config["embedding"]["show"] = True
+                if "allow_duplicates" in build_config:
+                    build_config["allow_duplicates"]["show"] = True
+                if "limit" in build_config:
+                    build_config["limit"]["show"] = True
+            elif field_value == "Retrieve":
+                if "persist" in build_config:
+                    build_config["persist"]["show"] = False
+                build_config["search_query"]["show"] = True
+                build_config["search_type"]["show"] = True
+                build_config["number_of_results"]["show"] = True
+                build_config["embedding"]["show"] = True
+                build_config["collection_name"]["show"] = False
+                # Show existing collections dropdown and update its options
+                if "existing_collections" in build_config:
+                    build_config["existing_collections"]["show"] = True
+                    build_config["existing_collections"]["options"] = self.list_existing_collections()
+                # Hide collection_name in Retrieve mode since we use existing_collections
+        elif field_name == "existing_collections":
+            # Update collection_name when an existing collection is selected
+            if "collection_name" in build_config:
+                build_config["collection_name"]["value"] = field_value
+
+        return build_config
+
+    @override
+    @check_cached_vector_store
+    def build_vector_store(self) -> Chroma:
+        """Builds the Chroma object."""
+        raise_error_if_astra_cloud_disable_component(disable_component_in_astra_cloud_msg)
+
+        try:
+            from langchain_chroma import Chroma
+        except ImportError as e:
+            msg = "Could not import Chroma integration package. Please install it with `pip install langchain-chroma`."
+            raise ImportError(msg) from e
+        # Chroma settings
+        # chroma_settings = None
+        if self.existing_collections:
+            self.collection_name = self.existing_collections
+
+        # Use user-provided directory or default cache directory
+        if self.persist_directory:
+            base_dir = self.resolve_path(self.persist_directory)
+            persist_directory = str(self.get_vector_store_directory(base_dir))
+            logger.debug(f"Using custom persist directory: {persist_directory}")
+        else:
+            persist_directory = self.get_default_persist_dir()
+            logger.debug(f"Using default persist directory: {persist_directory}")
+
+        chroma = Chroma(
+            persist_directory=persist_directory,
+            client=None,
+            embedding_function=self.embedding,
+            collection_name=self.collection_name,
+        )
+
+        self._add_documents_to_vector_store(chroma)
+        self.status = chroma_collection_to_data(chroma.get(limit=self.limit))
+        return chroma
+
+    def _add_documents_to_vector_store(self, vector_store: "Chroma") -> None:
+        """Adds documents to the Vector Store."""
+        ingest_data: list | Data | DataFrame = self.ingest_data
+        if not ingest_data:
+            self.status = ""
+            return
+
+        # Convert DataFrame to Data if needed using parent's method
+        ingest_data = self._prepare_ingest_data()
+
+        stored_documents_without_id = []
+        if self.allow_duplicates:
+            stored_data = []
+        else:
+            stored_data = chroma_collection_to_data(vector_store.get(limit=self.limit))
+            for value in deepcopy(stored_data):
+                del value.id
+                stored_documents_without_id.append(value)
+
+        documents = []
+        for _input in ingest_data or []:
+            if isinstance(_input, Data):
+                if _input not in stored_documents_without_id:
+                    documents.append(_input.to_lc_document())
+            else:
+                msg = "Vector Store Inputs must be Data objects."
+                raise TypeError(msg)
+
+        if documents and self.embedding is not None:
+            self.log(f"Adding {len(documents)} documents to the Vector Store.")
+            vector_store.add_documents(documents)
+        else:
+            self.log("No documents to add to the Vector Store.")
+
+    def perform_search(self) -> DataFrame:
+        return DataFrame(self.search_documents())

lfx/components/vertexai/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .vertexai import ChatVertexAIComponent
+    from .vertexai_embeddings import VertexAIEmbeddingsComponent
+
+_dynamic_imports = {
+    "ChatVertexAIComponent": "vertexai",
+    "VertexAIEmbeddingsComponent": "vertexai_embeddings",
+}
+
+__all__ = [
+    "ChatVertexAIComponent",
+    "VertexAIEmbeddingsComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import vertexai components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/vertexai/vertexai.py

@@ -0,0 +1,71 @@
+from typing import cast
+
+from lfx.base.models.model import LCModelComponent
+from lfx.field_typing import LanguageModel
+from lfx.inputs.inputs import MessageTextInput
+from lfx.io import BoolInput, FileInput, FloatInput, IntInput, StrInput
+
+
+class ChatVertexAIComponent(LCModelComponent):
+    display_name = "Vertex AI"
+    description = "Generate text using Vertex AI LLMs."
+    icon = "VertexAI"
+    name = "VertexAiModel"
+
+    inputs = [
+        *LCModelComponent.get_base_inputs(),
+        FileInput(
+            name="credentials",
+            display_name="Credentials",
+            info="JSON credentials file. Leave empty to fallback to environment variables",
+            file_types=["json"],
+        ),
+        MessageTextInput(name="model_name", display_name="Model Name", value="gemini-1.5-pro"),
+        StrInput(name="project", display_name="Project", info="The project ID.", advanced=True),
+        StrInput(name="location", display_name="Location", value="us-central1", advanced=True),
+        IntInput(name="max_output_tokens", display_name="Max Output Tokens", advanced=True),
+        IntInput(name="max_retries", display_name="Max Retries", value=1, advanced=True),
+        FloatInput(name="temperature", value=0.0, display_name="Temperature"),
+        IntInput(name="top_k", display_name="Top K", advanced=True),
+        FloatInput(name="top_p", display_name="Top P", value=0.95, advanced=True),
+        BoolInput(name="verbose", display_name="Verbose", value=False, advanced=True),
+    ]
+
+    def build_model(self) -> LanguageModel:
+        try:
+            from langchain_google_vertexai import ChatVertexAI
+        except ImportError as e:
+            msg = "Please install the langchain-google-vertexai package to use the VertexAIEmbeddings component."
+            raise ImportError(msg) from e
+        location = self.location or None
+        if self.credentials:
+            from google.cloud import aiplatform
+            from google.oauth2 import service_account
+
+            credentials = service_account.Credentials.from_service_account_file(self.credentials)
+            project = self.project or credentials.project_id
+            # ChatVertexAI sometimes skip manual credentials initialization
+            aiplatform.init(
+                project=project,
+                location=location,
+                credentials=credentials,
+            )
+        else:
+            project = self.project or None
+            credentials = None
+
+        return cast(
+            "LanguageModel",
+            ChatVertexAI(
+                credentials=credentials,
+                location=location,
+                project=project,
+                max_output_tokens=self.max_output_tokens or None,
+                max_retries=self.max_retries,
+                model_name=self.model_name,
+                temperature=self.temperature,
+                top_k=self.top_k or None,
+                top_p=self.top_p,
+                verbose=self.verbose,
+            ),
+        )