lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/docling/docling_remote.py

@@ -0,0 +1,195 @@
+import base64
+import time
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+import httpx
+from docling_core.types.doc import DoclingDocument
+from pydantic import ValidationError
+
+from lfx.base.data import BaseFileComponent
+from lfx.inputs import IntInput, NestedDictInput, StrInput
+from lfx.inputs.inputs import FloatInput
+from lfx.schema import Data
+from lfx.utils.util import transform_localhost_url
+
+
+class DoclingRemoteComponent(BaseFileComponent):
+    display_name = "Docling Serve"
+    description = "Uses Docling to process input documents connecting to your instance of Docling Serve."
+    documentation = "https://docling-project.github.io/docling/"
+    trace_type = "tool"
+    icon = "Docling"
+    name = "DoclingRemote"
+
+    MAX_500_RETRIES = 5
+
+    # https://docling-project.github.io/docling/usage/supported_formats/
+    VALID_EXTENSIONS = [
+        "adoc",
+        "asciidoc",
+        "asc",
+        "bmp",
+        "csv",
+        "dotx",
+        "dotm",
+        "docm",
+        "docx",
+        "htm",
+        "html",
+        "jpeg",
+        "json",
+        "md",
+        "pdf",
+        "png",
+        "potx",
+        "ppsx",
+        "pptm",
+        "potm",
+        "ppsm",
+        "pptx",
+        "tiff",
+        "txt",
+        "xls",
+        "xlsx",
+        "xhtml",
+        "xml",
+        "webp",
+    ]
+
+    inputs = [
+        *BaseFileComponent.get_base_inputs(),
+        StrInput(
+            name="api_url",
+            display_name="Server address",
+            info="URL of the Docling Serve instance.",
+            required=True,
+        ),
+        IntInput(
+            name="max_concurrency",
+            display_name="Concurrency",
+            info="Maximum number of concurrent requests for the server.",
+            advanced=True,
+            value=2,
+        ),
+        FloatInput(
+            name="max_poll_timeout",
+            display_name="Maximum poll time",
+            info="Maximum waiting time for the document conversion to complete.",
+            advanced=True,
+            value=3600,
+        ),
+        NestedDictInput(
+            name="api_headers",
+            display_name="HTTP headers",
+            advanced=True,
+            required=False,
+            info=("Optional dictionary of additional headers required for connecting to Docling Serve."),
+        ),
+        NestedDictInput(
+            name="docling_serve_opts",
+            display_name="Docling options",
+            advanced=True,
+            required=False,
+            info=(
+                "Optional dictionary of additional options. "
+                "See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information."
+            ),
+        ),
+    ]
+
+    outputs = [
+        *BaseFileComponent.get_base_outputs(),
+    ]
+
+    def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:
+        # Transform localhost URLs to container-accessible hosts when running in a container
+        transformed_url = transform_localhost_url(self.api_url)
+        base_url = f"{transformed_url}/v1"
+
+        def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:
+            encoded_doc = base64.b64encode(file_path.read_bytes()).decode()
+            payload = {
+                "options": options,
+                "sources": [{"kind": "file", "base64_string": encoded_doc, "filename": file_path.name}],
+            }
+
+            response = client.post(f"{base_url}/convert/source/async", json=payload)
+            response.raise_for_status()
+            task = response.json()
+
+            http_failures = 0
+            retry_status_start = 500
+            retry_status_end = 600
+            start_wait_time = time.monotonic()
+            while task["task_status"] not in ("success", "failure"):
+                # Check if processing exceeds the maximum poll timeout
+                processing_time = time.monotonic() - start_wait_time
+                if processing_time >= self.max_poll_timeout:
+                    msg = (
+                        f"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}."
+                        "Please increase the max_poll_timeout parameter or review why the processing "
+                        "takes long on the server."
+                    )
+                    self.log(msg)
+                    raise RuntimeError(msg)
+
+                # Call for a new status update
+                time.sleep(2)
+                response = client.get(f"{base_url}/status/poll/{task['task_id']}")
+
+                # Check if the status call gets into 5xx errors and retry
+                if retry_status_start <= response.status_code < retry_status_end:
+                    http_failures += 1
+                    if http_failures > self.MAX_500_RETRIES:
+                        self.log(f"The status requests got a http response {response.status_code} too many times.")
+                        return None
+                    continue
+
+                # Update task status
+                task = response.json()
+
+            result_resp = client.get(f"{base_url}/result/{task['task_id']}")
+            result_resp.raise_for_status()
+            result = result_resp.json()
+
+            if "json_content" not in result["document"] or result["document"]["json_content"] is None:
+                self.log("No JSON DoclingDocument found in the result.")
+                return None
+
+            try:
+                doc = DoclingDocument.model_validate(result["document"]["json_content"])
+                return Data(data={"doc": doc, "file_path": str(file_path)})
+            except ValidationError as e:
+                self.log(f"Error validating the document. {e}")
+                return None
+
+        docling_options = {
+            "to_formats": ["json"],
+            "image_export_mode": "placeholder",
+            **(self.docling_serve_opts or {}),
+        }
+
+        processed_data: list[Data | None] = []
+        with (
+            httpx.Client(headers=self.api_headers) as client,
+            ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,
+        ):
+            futures: list[tuple[int, Future]] = []
+            for i, file in enumerate(file_list):
+                if file.path is None:
+                    processed_data.append(None)
+                    continue
+
+                futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))
+
+            for _index, future in futures:
+                try:
+                    result_data = future.result()
+                    processed_data.append(result_data)
+                except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:
+                    self.log(f"Docling remote processing failed: {exc}")
+                    raise
+
+        return self.rollup_data(file_list, processed_data)

lfx/components/docling/export_docling_document.py

@@ -0,0 +1,117 @@
+from typing import Any
+
+from docling_core.types.doc import ImageRefMode
+
+from lfx.base.data.docling_utils import extract_docling_documents
+from lfx.custom import Component
+from lfx.io import DropdownInput, HandleInput, MessageTextInput, Output, StrInput
+from lfx.schema import Data, DataFrame
+
+
+class ExportDoclingDocumentComponent(Component):
+    display_name: str = "Export DoclingDocument"
+    description: str = "Export DoclingDocument to markdown, html or other formats."
+    documentation = "https://docling-project.github.io/docling/"
+    icon = "Docling"
+    name = "ExportDoclingDocument"
+
+    inputs = [
+        HandleInput(
+            name="data_inputs",
+            display_name="Data or DataFrame",
+            info="The data with documents to export.",
+            input_types=["Data", "DataFrame"],
+            required=True,
+        ),
+        DropdownInput(
+            name="export_format",
+            display_name="Export format",
+            options=["Markdown", "HTML", "Plaintext", "DocTags"],
+            info="Select the export format to convert the input.",
+            value="Markdown",
+            real_time_refresh=True,
+        ),
+        DropdownInput(
+            name="image_mode",
+            display_name="Image export mode",
+            options=["placeholder", "embedded"],
+            info=(
+                "Specify how images are exported in the output. Placeholder will replace the images with a string, "
+                "whereas Embedded will include them as base64 encoded images."
+            ),
+            value="placeholder",
+        ),
+        StrInput(
+            name="md_image_placeholder",
+            display_name="Image placeholder",
+            info="Specify the image placeholder for markdown exports.",
+            value="<!-- image -->",
+            advanced=True,
+        ),
+        StrInput(
+            name="md_page_break_placeholder",
+            display_name="Page break placeholder",
+            info="Add this placeholder betweek pages in the markdown output.",
+            value="",
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="doc_key",
+            display_name="Doc Key",
+            info="The key to use for the DoclingDocument column.",
+            value="doc",
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Exported data", name="data", method="export_document"),
+        Output(display_name="DataFrame", name="dataframe", method="as_dataframe"),
+    ]
+
+    def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:
+        if field_name == "export_format" and field_value == "Markdown":
+            build_config["md_image_placeholder"]["show"] = True
+            build_config["md_page_break_placeholder"]["show"] = True
+            build_config["image_mode"]["show"] = True
+        elif field_name == "export_format" and field_value == "HTML":
+            build_config["md_image_placeholder"]["show"] = False
+            build_config["md_page_break_placeholder"]["show"] = False
+            build_config["image_mode"]["show"] = True
+        elif field_name == "export_format" and field_value in {"Plaintext", "DocTags"}:
+            build_config["md_image_placeholder"]["show"] = False
+            build_config["md_page_break_placeholder"]["show"] = False
+            build_config["image_mode"]["show"] = False
+
+        return build_config
+
+    def export_document(self) -> list[Data]:
+        documents = extract_docling_documents(self.data_inputs, self.doc_key)
+
+        results: list[Data] = []
+        try:
+            image_mode = ImageRefMode(self.image_mode)
+            for doc in documents:
+                content = ""
+                if self.export_format == "Markdown":
+                    content = doc.export_to_markdown(
+                        image_mode=image_mode,
+                        image_placeholder=self.md_image_placeholder,
+                        page_break_placeholder=self.md_page_break_placeholder,
+                    )
+                elif self.export_format == "HTML":
+                    content = doc.export_to_html(image_mode=image_mode)
+                elif self.export_format == "Plaintext":
+                    content = doc.export_to_text()
+                elif self.export_format == "DocTags":
+                    content = doc.export_to_doctags()
+
+                results.append(Data(text=content))
+        except Exception as e:
+            msg = f"Error splitting text: {e}"
+            raise TypeError(msg) from e
+
+        return results
+
+    def as_dataframe(self) -> DataFrame:
+        return DataFrame(self.export_document())

lfx/components/documentloaders/__init__.py

@@ -0,0 +1,3 @@
+"""LangFlow document loaders components."""
+
+__all__: list[str] = []

lfx/components/duckduckgo/__init__.py

@@ -0,0 +1,3 @@
+from .duck_duck_go_search_run import DuckDuckGoSearchComponent
+
+__all__ = ["DuckDuckGoSearchComponent"]

lfx/components/duckduckgo/duck_duck_go_search_run.py

@@ -0,0 +1,92 @@
+from langchain_community.tools import DuckDuckGoSearchRun
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import IntInput, MessageTextInput
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.template.field.base import Output
+
+
+class DuckDuckGoSearchComponent(Component):
+    """Component for performing web searches using DuckDuckGo."""
+
+    display_name = "DuckDuckGo Search"
+    description = "Search the web using DuckDuckGo with customizable result limits"
+    documentation = "https://python.langchain.com/docs/integrations/tools/ddg"
+    icon = "DuckDuckGo"
+
+    inputs = [
+        MessageTextInput(
+            name="input_value",
+            display_name="Search Query",
+            required=True,
+            info="The search query to execute with DuckDuckGo",
+            tool_mode=True,
+        ),
+        IntInput(
+            name="max_results",
+            display_name="Max Results",
+            value=5,
+            required=False,
+            advanced=True,
+            info="Maximum number of search results to return",
+        ),
+        IntInput(
+            name="max_snippet_length",
+            display_name="Max Snippet Length",
+            value=100,
+            required=False,
+            advanced=True,
+            info="Maximum length of each result snippet",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="DataFrame", name="dataframe", method="fetch_content_dataframe"),
+    ]
+
+    def _build_wrapper(self) -> DuckDuckGoSearchRun:
+        """Build the DuckDuckGo search wrapper."""
+        return DuckDuckGoSearchRun()
+
+    def run_model(self) -> DataFrame:
+        return self.fetch_content_dataframe()
+
+    def fetch_content(self) -> list[Data]:
+        """Execute the search and return results as Data objects."""
+        try:
+            wrapper = self._build_wrapper()
+
+            full_results = wrapper.run(f"{self.input_value} (site:*)")
+
+            result_list = full_results.split("\n")[: self.max_results]
+
+            data_results = []
+            for result in result_list:
+                if result.strip():
+                    snippet = result[: self.max_snippet_length]
+                    data_results.append(
+                        Data(
+                            text=snippet,
+                            data={
+                                "content": result,
+                                "snippet": snippet,
+                            },
+                        )
+                    )
+        except (ValueError, AttributeError) as e:
+            error_data = [Data(text=str(e), data={"error": str(e)})]
+            self.status = error_data
+            return error_data
+        else:
+            self.status = data_results
+            return data_results
+
+    def fetch_content_dataframe(self) -> DataFrame:
+        """Convert the search results to a DataFrame.
+
+        Returns:
+            DataFrame: A DataFrame containing the search results.
+        """
+        data = self.fetch_content()
+        return DataFrame(data)

lfx/components/elastic/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .elasticsearch import ElasticsearchVectorStoreComponent
+    from .opensearch import OpenSearchVectorStoreComponent
+
+_dynamic_imports = {
+    "ElasticsearchVectorStoreComponent": "elasticsearch",
+    "OpenSearchVectorStoreComponent": "opensearch",
+}
+
+__all__ = [
+    "ElasticsearchVectorStoreComponent",
+    "OpenSearchVectorStoreComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import Elastic components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)