lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/unstructured/unstructured.py

@@ -0,0 +1,121 @@
+from langchain_unstructured import UnstructuredLoader
+
+from lfx.base.data.base_file import BaseFileComponent
+from lfx.inputs.inputs import DropdownInput, MessageTextInput, NestedDictInput, SecretStrInput
+from lfx.schema.data import Data
+
+
+class UnstructuredComponent(BaseFileComponent):
+    display_name = "Unstructured API"
+    description = (
+        "Uses Unstructured.io API to extract clean text from raw source documents. Supports a wide range of file types."
+    )
+    documentation = (
+        "https://python.langchain.com/api_reference/unstructured/document_loaders/"
+        "langchain_unstructured.document_loaders.UnstructuredLoader.html"
+    )
+    trace_type = "tool"
+    icon = "Unstructured"
+    name = "Unstructured"
+
+    # https://docs.unstructured.io/api-reference/api-services/overview#supported-file-types
+    VALID_EXTENSIONS = [
+        "bmp",
+        "csv",
+        "doc",
+        "docx",
+        "eml",
+        "epub",
+        "heic",
+        "html",
+        "jpeg",
+        "png",
+        "md",
+        "msg",
+        "odt",
+        "org",
+        "p7s",
+        "pdf",
+        "png",
+        "ppt",
+        "pptx",
+        "rst",
+        "rtf",
+        "tiff",
+        "txt",
+        "tsv",
+        "xls",
+        "xlsx",
+        "xml",
+    ]
+
+    inputs = [
+        *BaseFileComponent.get_base_inputs(),
+        SecretStrInput(
+            name="api_key",
+            display_name="Unstructured.io Serverless API Key",
+            required=True,
+            info="Unstructured API Key. Create at: https://app.unstructured.io/",
+        ),
+        MessageTextInput(
+            name="api_url",
+            display_name="Unstructured.io API URL",
+            required=False,
+            info="Unstructured API URL.",
+        ),
+        DropdownInput(
+            name="chunking_strategy",
+            display_name="Chunking Strategy",
+            info="Chunking strategy to use, see https://docs.unstructured.io/api-reference/api-services/chunking",
+            options=["", "basic", "by_title", "by_page", "by_similarity"],
+            real_time_refresh=False,
+            value="",
+        ),
+        NestedDictInput(
+            name="unstructured_args",
+            display_name="Additional Arguments",
+            required=False,
+            info=(
+                "Optional dictionary of additional arguments to the Loader. "
+                "See https://docs.unstructured.io/api-reference/api-services/api-parameters for more information."
+            ),
+        ),
+    ]
+
+    outputs = [
+        *BaseFileComponent.get_base_outputs(),
+    ]
+
+    def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:
+        file_paths = [str(file.path) for file in file_list if file.path]
+
+        if not file_paths:
+            self.log("No files to process.")
+            return file_list
+
+        # https://docs.unstructured.io/api-reference/api-services/api-parameters
+        args = self.unstructured_args or {}
+
+        if self.chunking_strategy:
+            args["chunking_strategy"] = self.chunking_strategy
+
+        args["api_key"] = self.api_key
+        args["partition_via_api"] = True
+        if self.api_url:
+            args["url"] = self.api_url
+
+        loader = UnstructuredLoader(
+            file_paths,
+            **args,
+        )
+
+        documents = loader.load()
+
+        processed_data: list[Data | None] = [Data.from_document(doc) if doc else None for doc in documents]
+
+        # Rename the `source` field to `self.SERVER_FILE_PATH_FIELDNAME`, to avoid conflicts with the `source` field
+        for data in processed_data:
+            if data and "source" in data.data:
+                data.data[self.SERVER_FILE_PATH_FIELDNAME] = data.data.pop("source")
+
+        return self.rollup_data(file_list, processed_data)

lfx/components/upstash/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .upstash import UpstashVectorStoreComponent
+
+_dynamic_imports = {
+    "UpstashVectorStoreComponent": "upstash",
+}
+
+__all__ = [
+    "UpstashVectorStoreComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import Upstash components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/upstash/upstash.py

@@ -0,0 +1,124 @@
+from langchain_community.vectorstores import UpstashVectorStore
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import (
+    HandleInput,
+    IntInput,
+    MultilineInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class UpstashVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "Upstash"
+    description = "Upstash Vector Store with search capabilities"
+    name = "Upstash"
+    icon = "Upstash"
+
+    inputs = [
+        StrInput(
+            name="index_url",
+            display_name="Index URL",
+            info="The URL of the Upstash index.",
+            required=True,
+        ),
+        SecretStrInput(
+            name="index_token",
+            display_name="Index Token",
+            info="The token for the Upstash index.",
+            required=True,
+        ),
+        StrInput(
+            name="text_key",
+            display_name="Text Key",
+            info="The key in the record to use as text.",
+            value="text",
+            advanced=True,
+        ),
+        StrInput(
+            name="namespace",
+            display_name="Namespace",
+            info="Leave empty for default namespace.",
+        ),
+        *LCVectorStoreComponent.inputs,
+        MultilineInput(
+            name="metadata_filter",
+            display_name="Metadata Filter",
+            info="Filters documents by metadata. Look at the documentation for more information.",
+        ),
+        HandleInput(
+            name="embedding",
+            display_name="Embedding",
+            input_types=["Embeddings"],
+            info="To use Upstash's embeddings, don't provide an embedding.",
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> UpstashVectorStore:
+        use_upstash_embedding = self.embedding is None
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents:
+            if use_upstash_embedding:
+                upstash_vs = UpstashVectorStore(
+                    embedding=use_upstash_embedding,
+                    text_key=self.text_key,
+                    index_url=self.index_url,
+                    index_token=self.index_token,
+                    namespace=self.namespace,
+                )
+                upstash_vs.add_documents(documents)
+            else:
+                upstash_vs = UpstashVectorStore.from_documents(
+                    documents=documents,
+                    embedding=self.embedding,
+                    text_key=self.text_key,
+                    index_url=self.index_url,
+                    index_token=self.index_token,
+                    namespace=self.namespace,
+                )
+        else:
+            upstash_vs = UpstashVectorStore(
+                embedding=self.embedding or use_upstash_embedding,
+                text_key=self.text_key,
+                index_url=self.index_url,
+                index_token=self.index_token,
+                namespace=self.namespace,
+            )
+
+        return upstash_vs
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+                filter=self.metadata_filter,
+            )
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []

lfx/components/vectara/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .vectara import VectaraVectorStoreComponent
+    from .vectara_rag import VectaraRagComponent
+
+_dynamic_imports = {
+    "VectaraVectorStoreComponent": "vectara",
+    "VectaraRagComponent": "vectara_rag",
+}
+
+__all__ = [
+    "VectaraRagComponent",
+    "VectaraVectorStoreComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import Vectara components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/vectara/vectara.py

@@ -0,0 +1,97 @@
+from typing import TYPE_CHECKING
+
+from langchain_community.vectorstores import Vectara
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+if TYPE_CHECKING:
+    from lfx.schema.dataframe import DataFrame
+
+
+class VectaraVectorStoreComponent(LCVectorStoreComponent):
+    """Vectara Vector Store with search capabilities."""
+
+    display_name: str = "Vectara"
+    description: str = "Vectara Vector Store with search capabilities"
+    name = "Vectara"
+    icon = "Vectara"
+
+    inputs = [
+        StrInput(name="vectara_customer_id", display_name="Vectara Customer ID", required=True),
+        StrInput(name="vectara_corpus_id", display_name="Vectara Corpus ID", required=True),
+        SecretStrInput(name="vectara_api_key", display_name="Vectara API Key", required=True),
+        HandleInput(
+            name="embedding",
+            display_name="Embedding",
+            input_types=["Embeddings"],
+        ),
+        *LCVectorStoreComponent.inputs,
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> Vectara:
+        """Builds the Vectara object."""
+        try:
+            from langchain_community.vectorstores import Vectara
+        except ImportError as e:
+            msg = "Could not import Vectara. Please install it with `pip install langchain-community`."
+            raise ImportError(msg) from e
+
+        vectara = Vectara(
+            vectara_customer_id=self.vectara_customer_id,
+            vectara_corpus_id=self.vectara_corpus_id,
+            vectara_api_key=self.vectara_api_key,
+        )
+
+        self._add_documents_to_vector_store(vectara)
+        return vectara
+
+    def _add_documents_to_vector_store(self, vector_store: Vectara) -> None:
+        """Adds documents to the Vector Store."""
+        ingest_data: list | Data | DataFrame = self.ingest_data
+        if not ingest_data:
+            self.status = "No documents to add to Vectara"
+            return
+
+        # Convert DataFrame to Data if needed using parent's method
+        ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents:
+            self.log(f"Adding {len(documents)} documents to Vectara.")
+            vector_store.add_documents(documents)
+            self.status = f"Added {len(documents)} documents to Vectara"
+        else:
+            self.log("No documents to add to Vectara.")
+            self.status = "No valid documents to add to Vectara"
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+
+            data = docs_to_data(docs)
+            self.status = f"Found {len(data)} results for the query: {self.search_query}"
+            return data
+        self.status = "No search query provided"
+        return []

lfx/components/vectara/vectara_rag.py

@@ -0,0 +1,164 @@
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing.range_spec import RangeSpec
+from lfx.io import DropdownInput, FloatInput, IntInput, MessageTextInput, Output, SecretStrInput, StrInput
+from lfx.schema.message import Message
+
+
+class VectaraRagComponent(Component):
+    display_name = "Vectara RAG"
+    description = "Vectara's full end to end RAG"
+    documentation = "https://docs.vectara.com/docs"
+    icon = "Vectara"
+    name = "VectaraRAG"
+    SUMMARIZER_PROMPTS = [
+        "vectara-summary-ext-24-05-sml",
+        "vectara-summary-ext-24-05-med-omni",
+        "vectara-summary-ext-24-05-large",
+        "vectara-summary-ext-24-05-med",
+        "vectara-summary-ext-v1.3.0",
+    ]
+
+    RERANKER_TYPES = ["mmr", "rerank_multilingual_v1", "none"]
+
+    RESPONSE_LANGUAGES = [
+        "auto",
+        "eng",
+        "spa",
+        "fra",
+        "zho",
+        "deu",
+        "hin",
+        "ara",
+        "por",
+        "ita",
+        "jpn",
+        "kor",
+        "rus",
+        "tur",
+        "fas",
+        "vie",
+        "tha",
+        "heb",
+        "nld",
+        "ind",
+        "pol",
+        "ukr",
+        "ron",
+        "swe",
+        "ces",
+        "ell",
+        "ben",
+        "msa",
+        "urd",
+    ]
+
+    field_order = ["vectara_customer_id", "vectara_corpus_id", "vectara_api_key", "search_query", "reranker"]
+
+    inputs = [
+        StrInput(name="vectara_customer_id", display_name="Vectara Customer ID", required=True),
+        StrInput(name="vectara_corpus_id", display_name="Vectara Corpus ID", required=True),
+        SecretStrInput(name="vectara_api_key", display_name="Vectara API Key", required=True),
+        MessageTextInput(
+            name="search_query",
+            display_name="Search Query",
+            info="The query to receive an answer on.",
+            tool_mode=True,
+        ),
+        FloatInput(
+            name="lexical_interpolation",
+            display_name="Hybrid Search Factor",
+            range_spec=RangeSpec(min=0.005, max=0.1, step=0.005),
+            value=0.005,
+            advanced=True,
+            info="How much to weigh lexical scores compared to the embedding score. "
+            "0 means lexical search is not used at all, and 1 means only lexical search is used.",
+        ),
+        MessageTextInput(
+            name="filter",
+            display_name="Metadata Filters",
+            value="",
+            advanced=True,
+            info="The filter string to narrow the search to according to metadata attributes.",
+        ),
+        DropdownInput(
+            name="reranker",
+            display_name="Reranker Type",
+            options=RERANKER_TYPES,
+            value=RERANKER_TYPES[0],
+            info="How to rerank the retrieved search results.",
+        ),
+        IntInput(
+            name="reranker_k",
+            display_name="Number of Results to Rerank",
+            value=50,
+            range_spec=RangeSpec(min=1, max=100, step=1),
+            advanced=True,
+        ),
+        FloatInput(
+            name="diversity_bias",
+            display_name="Diversity Bias",
+            value=0.2,
+            range_spec=RangeSpec(min=0, max=1, step=0.01),
+            advanced=True,
+            info="Ranges from 0 to 1, with higher values indicating greater diversity (only applies to MMR reranker).",
+        ),
+        IntInput(
+            name="max_results",
+            display_name="Max Results to Summarize",
+            value=7,
+            range_spec=RangeSpec(min=1, max=100, step=1),
+            advanced=True,
+            info="The maximum number of search results to be available to the prompt.",
+        ),
+        DropdownInput(
+            name="response_lang",
+            display_name="Response Language",
+            options=RESPONSE_LANGUAGES,
+            value="eng",
+            advanced=True,
+            info="Use the ISO 639-1 or 639-3 language code or auto to automatically detect the language.",
+        ),
+        DropdownInput(
+            name="prompt",
+            display_name="Prompt Name",
+            options=SUMMARIZER_PROMPTS,
+            value=SUMMARIZER_PROMPTS[0],
+            advanced=True,
+            info="Only vectara-summary-ext-24-05-sml is for Growth customers; "
+            "all other prompts are for Scale customers only.",
+        ),
+    ]
+
+    outputs = [
+        Output(name="answer", display_name="Answer", method="generate_response"),
+    ]
+
+    def generate_response(
+        self,
+    ) -> Message:
+        text_output = ""
+
+        try:
+            from langchain_community.vectorstores import Vectara
+            from langchain_community.vectorstores.vectara import RerankConfig, SummaryConfig, VectaraQueryConfig
+        except ImportError as e:
+            msg = "Could not import Vectara. Please install it with `pip install langchain-community`."
+            raise ImportError(msg) from e
+
+        vectara = Vectara(self.vectara_customer_id, self.vectara_corpus_id, self.vectara_api_key)
+        rerank_config = RerankConfig(self.reranker, self.reranker_k, self.diversity_bias)
+        summary_config = SummaryConfig(
+            is_enabled=True, max_results=self.max_results, response_lang=self.response_lang, prompt_name=self.prompt
+        )
+        config = VectaraQueryConfig(
+            lambda_val=self.lexical_interpolation,
+            filter=self.filter,
+            summary_config=summary_config,
+            rerank_config=rerank_config,
+        )
+        rag = vectara.as_rag(config)
+        response = rag.invoke(self.search_query, config={"callbacks": self.get_langchain_callbacks()})
+
+        text_output = response["answer"]
+
+        return Message(text=text_output)

lfx/components/vectorstores/__init__.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .astradb import AstraDBVectorStoreComponent
+    from .local_db import LocalDBComponent
+    from .mongodb_atlas import MongoVectorStoreComponent
+
+_dynamic_imports = {
+    "LocalDBComponent": "local_db",
+    "AstraDBVectorStoreComponent": "astradb",
+    "MongoVectorStoreComponent": "mongodb_atlas",
+}
+
+__all__ = [
+    "AstraDBVectorStoreComponent",
+    "LocalDBComponent",
+    "MongoVectorStoreComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import vectorstore components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)