lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/docling/export_docling_document.py

@@ -0,0 +1,117 @@
+from typing import Any
+
+from docling_core.types.doc import ImageRefMode
+
+from lfx.base.data.docling_utils import extract_docling_documents
+from lfx.custom import Component
+from lfx.io import DropdownInput, HandleInput, MessageTextInput, Output, StrInput
+from lfx.schema import Data, DataFrame
+
+
+class ExportDoclingDocumentComponent(Component):
+    display_name: str = "Export DoclingDocument"
+    description: str = "Export DoclingDocument to markdown, html or other formats."
+    documentation = "https://docling-project.github.io/docling/"
+    icon = "Docling"
+    name = "ExportDoclingDocument"
+
+    inputs = [
+        HandleInput(
+            name="data_inputs",
+            display_name="Data or DataFrame",
+            info="The data with documents to export.",
+            input_types=["Data", "DataFrame"],
+            required=True,
+        ),
+        DropdownInput(
+            name="export_format",
+            display_name="Export format",
+            options=["Markdown", "HTML", "Plaintext", "DocTags"],
+            info="Select the export format to convert the input.",
+            value="Markdown",
+            real_time_refresh=True,
+        ),
+        DropdownInput(
+            name="image_mode",
+            display_name="Image export mode",
+            options=["placeholder", "embedded"],
+            info=(
+                "Specify how images are exported in the output. Placeholder will replace the images with a string, "
+                "whereas Embedded will include them as base64 encoded images."
+            ),
+            value="placeholder",
+        ),
+        StrInput(
+            name="md_image_placeholder",
+            display_name="Image placeholder",
+            info="Specify the image placeholder for markdown exports.",
+            value="<!-- image -->",
+            advanced=True,
+        ),
+        StrInput(
+            name="md_page_break_placeholder",
+            display_name="Page break placeholder",
+            info="Add this placeholder betweek pages in the markdown output.",
+            value="",
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="doc_key",
+            display_name="Doc Key",
+            info="The key to use for the DoclingDocument column.",
+            value="doc",
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Exported data", name="data", method="export_document"),
+        Output(display_name="DataFrame", name="dataframe", method="as_dataframe"),
+    ]
+
+    def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:
+        if field_name == "export_format" and field_value == "Markdown":
+            build_config["md_image_placeholder"]["show"] = True
+            build_config["md_page_break_placeholder"]["show"] = True
+            build_config["image_mode"]["show"] = True
+        elif field_name == "export_format" and field_value == "HTML":
+            build_config["md_image_placeholder"]["show"] = False
+            build_config["md_page_break_placeholder"]["show"] = False
+            build_config["image_mode"]["show"] = True
+        elif field_name == "export_format" and field_value in {"Plaintext", "DocTags"}:
+            build_config["md_image_placeholder"]["show"] = False
+            build_config["md_page_break_placeholder"]["show"] = False
+            build_config["image_mode"]["show"] = False
+
+        return build_config
+
+    def export_document(self) -> list[Data]:
+        documents = extract_docling_documents(self.data_inputs, self.doc_key)
+
+        results: list[Data] = []
+        try:
+            image_mode = ImageRefMode(self.image_mode)
+            for doc in documents:
+                content = ""
+                if self.export_format == "Markdown":
+                    content = doc.export_to_markdown(
+                        image_mode=image_mode,
+                        image_placeholder=self.md_image_placeholder,
+                        page_break_placeholder=self.md_page_break_placeholder,
+                    )
+                elif self.export_format == "HTML":
+                    content = doc.export_to_html(image_mode=image_mode)
+                elif self.export_format == "Plaintext":
+                    content = doc.export_to_text()
+                elif self.export_format == "DocTags":
+                    content = doc.export_to_doctags()
+
+                results.append(Data(text=content))
+        except Exception as e:
+            msg = f"Error splitting text: {e}"
+            raise TypeError(msg) from e
+
+        return results
+
+    def as_dataframe(self) -> DataFrame:
+        return DataFrame(self.export_document())

lfx/components/documentloaders/__init__.py

@@ -0,0 +1,3 @@
+"""LangFlow document loaders components."""
+
+__all__: list[str] = []

lfx/components/duckduckgo/__init__.py

@@ -0,0 +1,3 @@
+from .duck_duck_go_search_run import DuckDuckGoSearchComponent
+
+__all__ = ["DuckDuckGoSearchComponent"]

lfx/components/duckduckgo/duck_duck_go_search_run.py

@@ -0,0 +1,92 @@
+from langchain_community.tools import DuckDuckGoSearchRun
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import IntInput, MessageTextInput
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.template.field.base import Output
+
+
+class DuckDuckGoSearchComponent(Component):
+    """Component for performing web searches using DuckDuckGo."""
+
+    display_name = "DuckDuckGo Search"
+    description = "Search the web using DuckDuckGo with customizable result limits"
+    documentation = "https://python.langchain.com/docs/integrations/tools/ddg"
+    icon = "DuckDuckGo"
+
+    inputs = [
+        MessageTextInput(
+            name="input_value",
+            display_name="Search Query",
+            required=True,
+            info="The search query to execute with DuckDuckGo",
+            tool_mode=True,
+        ),
+        IntInput(
+            name="max_results",
+            display_name="Max Results",
+            value=5,
+            required=False,
+            advanced=True,
+            info="Maximum number of search results to return",
+        ),
+        IntInput(
+            name="max_snippet_length",
+            display_name="Max Snippet Length",
+            value=100,
+            required=False,
+            advanced=True,
+            info="Maximum length of each result snippet",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="DataFrame", name="dataframe", method="fetch_content_dataframe"),
+    ]
+
+    def _build_wrapper(self) -> DuckDuckGoSearchRun:
+        """Build the DuckDuckGo search wrapper."""
+        return DuckDuckGoSearchRun()
+
+    def run_model(self) -> DataFrame:
+        return self.fetch_content_dataframe()
+
+    def fetch_content(self) -> list[Data]:
+        """Execute the search and return results as Data objects."""
+        try:
+            wrapper = self._build_wrapper()
+
+            full_results = wrapper.run(f"{self.input_value} (site:*)")
+
+            result_list = full_results.split("\n")[: self.max_results]
+
+            data_results = []
+            for result in result_list:
+                if result.strip():
+                    snippet = result[: self.max_snippet_length]
+                    data_results.append(
+                        Data(
+                            text=snippet,
+                            data={
+                                "content": result,
+                                "snippet": snippet,
+                            },
+                        )
+                    )
+        except (ValueError, AttributeError) as e:
+            error_data = [Data(text=str(e), data={"error": str(e)})]
+            self.status = error_data
+            return error_data
+        else:
+            self.status = data_results
+            return data_results
+
+    def fetch_content_dataframe(self) -> DataFrame:
+        """Convert the search results to a DataFrame.
+
+        Returns:
+            DataFrame: A DataFrame containing the search results.
+        """
+        data = self.fetch_content()
+        return DataFrame(data)

lfx/components/elastic/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .elasticsearch import ElasticsearchVectorStoreComponent
+    from .opensearch import OpenSearchVectorStoreComponent
+
+_dynamic_imports = {
+    "ElasticsearchVectorStoreComponent": "elasticsearch",
+    "OpenSearchVectorStoreComponent": "opensearch",
+}
+
+__all__ = [
+    "ElasticsearchVectorStoreComponent",
+    "OpenSearchVectorStoreComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import Elastic components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/elastic/elasticsearch.py

@@ -0,0 +1,267 @@
+from typing import Any
+
+from elasticsearch import Elasticsearch
+from langchain.schema import Document
+from langchain_elasticsearch import ElasticsearchStore
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.io import (
+    BoolInput,
+    DropdownInput,
+    FloatInput,
+    HandleInput,
+    IntInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class ElasticsearchVectorStoreComponent(LCVectorStoreComponent):
+    """Elasticsearch Vector Store with with advanced, customizable search capabilities."""
+
+    display_name: str = "Elasticsearch"
+    description: str = "Elasticsearch Vector Store with with advanced, customizable search capabilities."
+    name = "Elasticsearch"
+    icon = "ElasticsearchStore"
+
+    inputs = [
+        StrInput(
+            name="elasticsearch_url",
+            display_name="Elasticsearch URL",
+            value="http://localhost:9200",
+            info="URL for self-managed Elasticsearch deployments (e.g., http://localhost:9200). "
+            "Do not use with Elastic Cloud deployments, use Elastic Cloud ID instead.",
+        ),
+        SecretStrInput(
+            name="cloud_id",
+            display_name="Elastic Cloud ID",
+            value="",
+            info="Use this for Elastic Cloud deployments. Do not use together with 'Elasticsearch URL'.",
+        ),
+        StrInput(
+            name="index_name",
+            display_name="Index Name",
+            value="langflow",
+            info="The index name where the vectors will be stored in Elasticsearch cluster.",
+        ),
+        *LCVectorStoreComponent.inputs,
+        StrInput(
+            name="username",
+            display_name="Username",
+            value="",
+            advanced=False,
+            info=(
+                "Elasticsearch username (e.g., 'elastic'). "
+                "Required for both local and Elastic Cloud setups unless API keys are used."
+            ),
+        ),
+        SecretStrInput(
+            name="password",
+            display_name="Password",
+            value="",
+            advanced=False,
+            info=(
+                "Elasticsearch password for the specified user. "
+                "Required for both local and Elastic Cloud setups unless API keys are used."
+            ),
+        ),
+        HandleInput(
+            name="embedding",
+            display_name="Embedding",
+            input_types=["Embeddings"],
+        ),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Type",
+            options=["similarity", "mmr"],
+            value="similarity",
+            advanced=True,
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=4,
+        ),
+        FloatInput(
+            name="search_score_threshold",
+            display_name="Search Score Threshold",
+            info="Minimum similarity score threshold for search results.",
+            value=0.0,
+            advanced=True,
+        ),
+        SecretStrInput(
+            name="api_key",
+            display_name="Elastic API Key",
+            value="",
+            advanced=True,
+            info="API Key for Elastic Cloud authentication. If used, 'username' and 'password' are not required.",
+        ),
+        BoolInput(
+            name="verify_certs",
+            display_name="Verify SSL Certificates",
+            value=True,
+            advanced=True,
+            info="Whether to verify SSL certificates when connecting to Elasticsearch.",
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> ElasticsearchStore:
+        """Builds the Elasticsearch Vector Store object."""
+        if self.cloud_id and self.elasticsearch_url:
+            msg = (
+                "Both 'cloud_id' and 'elasticsearch_url' provided. "
+                "Please use only one based on your deployment (Cloud or Local)."
+            )
+            raise ValueError(msg)
+
+        es_params = {
+            "index_name": self.index_name,
+            "embedding": self.embedding,
+            "es_user": self.username or None,
+            "es_password": self.password or None,
+        }
+
+        if self.cloud_id:
+            es_params["es_cloud_id"] = self.cloud_id
+        else:
+            es_params["es_url"] = self.elasticsearch_url
+
+        if self.api_key:
+            es_params["api_key"] = self.api_key
+
+        # Check if we need to verify SSL certificates
+        if self.verify_certs is False:
+            # Build client parameters for Elasticsearch constructor
+            client_params: dict[str, Any] = {}
+            client_params["verify_certs"] = False
+
+            if self.cloud_id:
+                client_params["cloud_id"] = self.cloud_id
+            else:
+                client_params["hosts"] = [self.elasticsearch_url]
+
+            if self.api_key:
+                client_params["api_key"] = self.api_key
+            elif self.username and self.password:
+                client_params["basic_auth"] = (self.username, self.password)
+
+            es_client = Elasticsearch(**client_params)
+            es_params["es_connection"] = es_client
+
+        elasticsearch = ElasticsearchStore(**es_params)
+
+        # If documents are provided, add them to the store
+        if self.ingest_data:
+            documents = self._prepare_documents()
+            if documents:
+                elasticsearch.add_documents(documents)
+
+        return elasticsearch
+
+    def _prepare_documents(self) -> list[Document]:
+        """Prepares documents from the input data to add to the vector store."""
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for data in self.ingest_data:
+            if isinstance(data, Data):
+                documents.append(data.to_lc_document())
+            else:
+                error_message = "Vector Store Inputs must be Data objects."
+                self.log(error_message)
+                raise TypeError(error_message)
+        return documents
+
+    def _add_documents_to_vector_store(self, vector_store: "ElasticsearchStore") -> None:
+        """Adds documents to the Vector Store."""
+        documents = self._prepare_documents()
+        if documents and self.embedding:
+            self.log(f"Adding {len(documents)} documents to the Vector Store.")
+            vector_store.add_documents(documents)
+        else:
+            self.log("No documents to add to the Vector Store.")
+
+    def search(self, query: str | None = None) -> list[dict[str, Any]]:
+        """Search for similar documents in the vector store or retrieve all documents if no query is provided."""
+        vector_store = self.build_vector_store()
+        search_kwargs = {
+            "k": self.number_of_results,
+            "score_threshold": self.search_score_threshold,
+        }
+
+        if query:
+            search_type = self.search_type.lower()
+            if search_type not in {"similarity", "mmr"}:
+                msg = f"Invalid search type: {self.search_type}"
+                self.log(msg)
+                raise ValueError(msg)
+            try:
+                if search_type == "similarity":
+                    results = vector_store.similarity_search_with_score(query, **search_kwargs)
+                elif search_type == "mmr":
+                    results = vector_store.max_marginal_relevance_search(query, **search_kwargs)
+            except Exception as e:
+                msg = (
+                    "Error occurred while querying the Elasticsearch VectorStore,"
+                    " there is no Data into the VectorStore."
+                )
+                self.log(msg)
+                raise ValueError(msg) from e
+            return [
+                {"page_content": doc.page_content, "metadata": doc.metadata, "score": score} for doc, score in results
+            ]
+        results = self.get_all_documents(vector_store, **search_kwargs)
+        return [{"page_content": doc.page_content, "metadata": doc.metadata, "score": score} for doc, score in results]
+
+    def get_all_documents(self, vector_store: ElasticsearchStore, **kwargs) -> list[tuple[Document, float]]:
+        """Retrieve all documents from the vector store."""
+        client = vector_store.client
+        index_name = self.index_name
+
+        query = {
+            "query": {"match_all": {}},
+            "size": kwargs.get("k", self.number_of_results),
+        }
+
+        response = client.search(index=index_name, body=query)
+
+        results = []
+        for hit in response["hits"]["hits"]:
+            doc = Document(
+                page_content=hit["_source"].get("text", ""),
+                metadata=hit["_source"].get("metadata", {}),
+            )
+            score = hit["_score"]
+            results.append((doc, score))
+
+        return results
+
+    def search_documents(self) -> list[Data]:
+        """Search for documents in the vector store based on the search input.
+
+        If no search input is provided, retrieve all documents.
+        """
+        results = self.search(self.search_query)
+        retrieved_data = [
+            Data(
+                text=result["page_content"],
+                file_path=result["metadata"].get("file_path", ""),
+            )
+            for result in results
+        ]
+        self.status = retrieved_data
+        return retrieved_data
+
+    def get_retriever_kwargs(self):
+        """Get the keyword arguments for the retriever."""
+        return {
+            "search_type": self.search_type.lower(),
+            "search_kwargs": {
+                "k": self.number_of_results,
+                "score_threshold": self.search_score_threshold,
+            },
+        }