lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/base/data/docling_utils.py

@@ -0,0 +1,245 @@
+import signal
+import sys
+import traceback
+from contextlib import suppress
+
+from docling_core.types.doc import DoclingDocument
+
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+
+
+def extract_docling_documents(data_inputs: Data | list[Data] | DataFrame, doc_key: str) -> list[DoclingDocument]:
+    documents: list[DoclingDocument] = []
+    if isinstance(data_inputs, DataFrame):
+        if not len(data_inputs):
+            msg = "DataFrame is empty"
+            raise TypeError(msg)
+
+        if doc_key not in data_inputs.columns:
+            msg = f"Column '{doc_key}' not found in DataFrame"
+            raise TypeError(msg)
+        try:
+            documents = data_inputs[doc_key].tolist()
+        except Exception as e:
+            msg = f"Error extracting DoclingDocument from DataFrame: {e}"
+            raise TypeError(msg) from e
+    else:
+        if not data_inputs:
+            msg = "No data inputs provided"
+            raise TypeError(msg)
+
+        if isinstance(data_inputs, Data):
+            if doc_key not in data_inputs.data:
+                msg = (
+                    f"'{doc_key}' field not available in the input Data. "
+                    "Check that your input is a DoclingDocument. "
+                    "You can use the Docling component to convert your input to a DoclingDocument."
+                )
+                raise TypeError(msg)
+            documents = [data_inputs.data[doc_key]]
+        else:
+            try:
+                documents = [
+                    input_.data[doc_key]
+                    for input_ in data_inputs
+                    if isinstance(input_, Data)
+                    and doc_key in input_.data
+                    and isinstance(input_.data[doc_key], DoclingDocument)
+                ]
+                if not documents:
+                    msg = f"No valid Data inputs found in {type(data_inputs)}"
+                    raise TypeError(msg)
+            except AttributeError as e:
+                msg = f"Invalid input type in collection: {e}"
+                raise TypeError(msg) from e
+    return documents
+
+
+def docling_worker(file_paths: list[str], queue, pipeline: str, ocr_engine: str):
+    """Worker function for processing files with Docling in a separate process."""
+    # Signal handling for graceful shutdown
+    shutdown_requested = False
+
+    def signal_handler(signum: int, frame) -> None:  # noqa: ARG001
+        """Handle shutdown signals gracefully."""
+        nonlocal shutdown_requested
+        signal_names: dict[int, str] = {signal.SIGTERM: "SIGTERM", signal.SIGINT: "SIGINT"}
+        signal_name = signal_names.get(signum, f"signal {signum}")
+
+        logger.debug(f"Docling worker received {signal_name}, initiating graceful shutdown...")
+        shutdown_requested = True
+
+        # Send shutdown notification to parent process
+        with suppress(Exception):
+            queue.put({"error": f"Worker interrupted by {signal_name}", "shutdown": True})
+
+        # Exit gracefully
+        sys.exit(0)
+
+    def check_shutdown() -> None:
+        """Check if shutdown was requested and exit if so."""
+        if shutdown_requested:
+            logger.info("Shutdown requested, exiting worker...")
+
+            with suppress(Exception):
+                queue.put({"error": "Worker shutdown requested", "shutdown": True})
+
+            sys.exit(0)
+
+    # Register signal handlers early
+    try:
+        signal.signal(signal.SIGTERM, signal_handler)
+        signal.signal(signal.SIGINT, signal_handler)
+        logger.debug("Signal handlers registered for graceful shutdown")
+    except (OSError, ValueError) as e:
+        # Some signals might not be available on all platforms
+        logger.warning(f"Warning: Could not register signal handlers: {e}")
+
+    # Check for shutdown before heavy imports
+    check_shutdown()
+
+    try:
+        from docling.datamodel.base_models import ConversionStatus, InputFormat
+        from docling.datamodel.pipeline_options import OcrOptions, PdfPipelineOptions, VlmPipelineOptions
+        from docling.document_converter import DocumentConverter, FormatOption, PdfFormatOption
+        from docling.models.factories import get_ocr_factory
+        from docling.pipeline.vlm_pipeline import VlmPipeline
+
+        # Check for shutdown after imports
+        check_shutdown()
+        logger.debug("Docling dependencies loaded successfully")
+
+    except ModuleNotFoundError:
+        msg = (
+            "Docling is an optional dependency of Langflow. "
+            "Install with `uv pip install 'langflow[docling]'` "
+            "or refer to the documentation"
+        )
+        queue.put({"error": msg})
+        return
+    except ImportError as e:
+        # A different import failed (e.g., a transitive dependency); preserve details.
+        queue.put({"error": f"Failed to import a Docling dependency: {e}"})
+        return
+    except KeyboardInterrupt:
+        logger.warning("KeyboardInterrupt during imports, exiting...")
+        queue.put({"error": "Worker interrupted during imports", "shutdown": True})
+        return
+
+    # Configure the standard PDF pipeline
+    def _get_standard_opts() -> PdfPipelineOptions:
+        check_shutdown()  # Check before heavy operations
+
+        pipeline_options = PdfPipelineOptions()
+        pipeline_options.do_ocr = ocr_engine != ""
+        if pipeline_options.do_ocr:
+            ocr_factory = get_ocr_factory(
+                allow_external_plugins=False,
+            )
+
+            ocr_options: OcrOptions = ocr_factory.create_options(
+                kind=ocr_engine,
+            )
+            pipeline_options.ocr_options = ocr_options
+        return pipeline_options
+
+    # Configure the VLM pipeline
+    def _get_vlm_opts() -> VlmPipelineOptions:
+        check_shutdown()  # Check before heavy operations
+        return VlmPipelineOptions()
+
+    # Configure the main format options and create the DocumentConverter()
+    def _get_converter() -> DocumentConverter:
+        check_shutdown()  # Check before heavy operations
+
+        if pipeline == "standard":
+            pdf_format_option = PdfFormatOption(
+                pipeline_options=_get_standard_opts(),
+            )
+        elif pipeline == "vlm":
+            pdf_format_option = PdfFormatOption(pipeline_cls=VlmPipeline, pipeline_options=_get_vlm_opts())
+        else:
+            msg = f"Unknown pipeline: {pipeline!r}"
+            raise ValueError(msg)
+
+        format_options: dict[InputFormat, FormatOption] = {
+            InputFormat.PDF: pdf_format_option,
+            InputFormat.IMAGE: pdf_format_option,
+        }
+
+        return DocumentConverter(format_options=format_options)
+
+    try:
+        # Check for shutdown before creating converter (can be slow)
+        check_shutdown()
+        logger.info(f"Initializing {pipeline} pipeline with OCR: {ocr_engine or 'disabled'}")
+
+        converter = _get_converter()
+
+        # Check for shutdown before processing files
+        check_shutdown()
+        logger.info(f"Starting to process {len(file_paths)} files...")
+
+        # Process files with periodic shutdown checks
+        results = []
+        for i, file_path in enumerate(file_paths):
+            # Check for shutdown before processing each file
+            check_shutdown()
+
+            logger.debug(f"Processing file {i + 1}/{len(file_paths)}: {file_path}")
+
+            try:
+                # Process single file (we can't easily interrupt convert_all)
+                single_result = converter.convert_all([file_path])
+                results.extend(single_result)
+
+                # Check for shutdown after each file
+                check_shutdown()
+
+            except (OSError, ValueError, RuntimeError, ImportError) as file_error:
+                # Handle specific file processing errors
+                logger.error(f"Error processing file {file_path}: {file_error}")
+                # Continue with other files, but check for shutdown
+                check_shutdown()
+            except Exception as file_error:  # noqa: BLE001
+                # Catch any other unexpected errors to prevent worker crash
+                logger.error(f"Unexpected error processing file {file_path}: {file_error}")
+                # Continue with other files, but check for shutdown
+                check_shutdown()
+
+        # Final shutdown check before sending results
+        check_shutdown()
+
+        # Process the results while maintaining the original structure
+        processed_data = [
+            {"document": res.document, "file_path": str(res.input.file), "status": res.status.name}
+            if res.status == ConversionStatus.SUCCESS
+            else None
+            for res in results
+        ]
+
+        logger.info(f"Successfully processed {len([d for d in processed_data if d])} files")
+        queue.put(processed_data)
+
+    except KeyboardInterrupt:
+        logger.warning("KeyboardInterrupt during processing, exiting gracefully...")
+        queue.put({"error": "Worker interrupted during processing", "shutdown": True})
+        return
+    except Exception as e:  # noqa: BLE001
+        if shutdown_requested:
+            logger.exception("Exception occurred during shutdown, exiting...")
+            return
+
+        # Send any processing error to the main process with traceback
+        error_info = {"error": str(e), "traceback": traceback.format_exc()}
+        logger.error(f"Error in worker: {error_info}")
+        queue.put(error_info)
+    finally:
+        logger.info("Docling worker finishing...")
+        # Ensure we don't leave any hanging processes
+        if shutdown_requested:
+            logger.debug("Worker shutdown completed")
+        else:
+            logger.debug("Worker completed normally")

lfx/base/data/utils.py

@@ -0,0 +1,198 @@
+import unicodedata
+from collections.abc import Callable
+from concurrent import futures
+from pathlib import Path
+
+import chardet
+import orjson
+import yaml
+from defusedxml import ElementTree
+
+from lfx.schema.data import Data
+
+# Types of files that can be read simply by file.read()
+# and have 100% to be completely readable
+TEXT_FILE_TYPES = [
+    "txt",
+    "md",
+    "mdx",
+    "csv",
+    "json",
+    "yaml",
+    "yml",
+    "xml",
+    "html",
+    "htm",
+    "pdf",
+    "docx",
+    "py",
+    "sh",
+    "sql",
+    "js",
+    "ts",
+    "tsx",
+]
+
+IMG_FILE_TYPES = ["jpg", "jpeg", "png", "bmp", "image"]
+
+
+def normalize_text(text):
+    return unicodedata.normalize("NFKD", text)
+
+
+def is_hidden(path: Path) -> bool:
+    return path.name.startswith(".")
+
+
+def format_directory_path(path: str) -> str:
+    """Format a directory path to ensure it's properly escaped and valid.
+
+    Args:
+    path (str): The input path string.
+
+    Returns:
+    str: A properly formatted path string.
+    """
+    return path.replace("\n", "\\n")
+
+
+# Ignoring FBT001 because the DirectoryComponent in 1.0.19
+# calls this function without keyword arguments
+def retrieve_file_paths(
+    path: str,
+    load_hidden: bool,  # noqa: FBT001
+    recursive: bool,  # noqa: FBT001
+    depth: int,
+    types: list[str] = TEXT_FILE_TYPES,
+) -> list[str]:
+    path = format_directory_path(path)
+    path_obj = Path(path)
+    if not path_obj.exists() or not path_obj.is_dir():
+        msg = f"Path {path} must exist and be a directory."
+        raise ValueError(msg)
+
+    def match_types(p: Path) -> bool:
+        return any(p.suffix == f".{t}" for t in types) if types else True
+
+    def is_not_hidden(p: Path) -> bool:
+        return not is_hidden(p) or load_hidden
+
+    def walk_level(directory: Path, max_depth: int):
+        directory = directory.resolve()
+        prefix_length = len(directory.parts)
+        for p in directory.rglob("*" if recursive else "[!.]*"):
+            if len(p.parts) - prefix_length <= max_depth:
+                yield p
+
+    glob = "**/*" if recursive else "*"
+    paths = walk_level(path_obj, depth) if depth else path_obj.glob(glob)
+    return [str(p) for p in paths if p.is_file() and match_types(p) and is_not_hidden(p)]
+
+
+def partition_file_to_data(file_path: str, *, silent_errors: bool) -> Data | None:
+    # Use the partition function to load the file
+    from unstructured.partition.auto import partition
+
+    try:
+        elements = partition(file_path)
+    except Exception as e:
+        if not silent_errors:
+            msg = f"Error loading file {file_path}: {e}"
+            raise ValueError(msg) from e
+        return None
+
+    # Create a Data
+    text = "\n\n".join([str(el) for el in elements])
+    metadata = elements.metadata if hasattr(elements, "metadata") else {}
+    metadata["file_path"] = file_path
+    return Data(text=text, data=metadata)
+
+
+def read_text_file(file_path: str) -> str:
+    file_path_ = Path(file_path)
+    raw_data = file_path_.read_bytes()
+    result = chardet.detect(raw_data)
+    encoding = result["encoding"]
+
+    if encoding in {"Windows-1252", "Windows-1254", "MacRoman"}:
+        encoding = "utf-8"
+
+    return file_path_.read_text(encoding=encoding)
+
+
+def read_docx_file(file_path: str) -> str:
+    from docx import Document
+
+    doc = Document(file_path)
+    return "\n\n".join([p.text for p in doc.paragraphs])
+
+
+def parse_pdf_to_text(file_path: str) -> str:
+    from pypdf import PdfReader
+
+    with Path(file_path).open("rb") as f, PdfReader(f) as reader:
+        return "\n\n".join([page.extract_text() for page in reader.pages])
+
+
+def parse_text_file_to_data(file_path: str, *, silent_errors: bool) -> Data | None:
+    try:
+        if file_path.endswith(".pdf"):
+            text = parse_pdf_to_text(file_path)
+        elif file_path.endswith(".docx"):
+            text = read_docx_file(file_path)
+        else:
+            text = read_text_file(file_path)
+
+        # if file is json, yaml, or xml, we can parse it
+        if file_path.endswith(".json"):
+            loaded_json = orjson.loads(text)
+            if isinstance(loaded_json, dict):
+                loaded_json = {k: normalize_text(v) if isinstance(v, str) else v for k, v in loaded_json.items()}
+            elif isinstance(loaded_json, list):
+                loaded_json = [normalize_text(item) if isinstance(item, str) else item for item in loaded_json]
+            text = orjson.dumps(loaded_json).decode("utf-8")
+
+        elif file_path.endswith((".yaml", ".yml")):
+            text = yaml.safe_load(text)
+        elif file_path.endswith(".xml"):
+            xml_element = ElementTree.fromstring(text)
+            text = ElementTree.tostring(xml_element, encoding="unicode")
+    except Exception as e:
+        if not silent_errors:
+            msg = f"Error loading file {file_path}: {e}"
+            raise ValueError(msg) from e
+        return None
+
+    return Data(data={"file_path": file_path, "text": text})
+
+
+# ! Removing unstructured dependency until
+# ! 3.12 is supported
+# def get_elements(
+#     file_paths: List[str],
+#     silent_errors: bool,
+#     max_concurrency: int,
+#     use_multithreading: bool,
+# ) -> List[Optional[Data]]:
+#     if use_multithreading:
+#         data = parallel_load_data(file_paths, silent_errors, max_concurrency)
+#     else:
+#         data = [partition_file_to_data(file_path, silent_errors) for file_path in file_paths]
+#     data = list(filter(None, data))
+#     return data
+
+
+def parallel_load_data(
+    file_paths: list[str],
+    *,
+    silent_errors: bool,
+    max_concurrency: int,
+    load_function: Callable = parse_text_file_to_data,
+) -> list[Data | None]:
+    with futures.ThreadPoolExecutor(max_workers=max_concurrency) as executor:
+        loaded_files = executor.map(
+            lambda file_path: load_function(file_path, silent_errors=silent_errors),
+            file_paths,
+        )
+    # loaded_files is an iterator, so we need to convert it to a list
+    return list(loaded_files)

lfx/base/document_transformers/model.py

@@ -0,0 +1,43 @@
+from abc import abstractmethod
+from typing import Any
+
+from langchain_core.documents import BaseDocumentTransformer
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import Output
+from lfx.schema.data import Data
+from lfx.utils.util import build_loader_repr_from_data
+
+
+class LCDocumentTransformerComponent(Component):
+    trace_type = "document_transformer"
+    outputs = [
+        Output(display_name="Data", name="data", method="transform_data"),
+    ]
+
+    def transform_data(self) -> list[Data]:
+        data_input = self.get_data_input()
+        documents = []
+
+        if not isinstance(data_input, list):
+            data_input = [data_input]
+
+        for _input in data_input:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        transformer = self.build_document_transformer()
+        docs = transformer.transform_documents(documents)
+        data = self.to_data(docs)
+        self.repr_value = build_loader_repr_from_data(data)
+        return data
+
+    @abstractmethod
+    def get_data_input(self) -> Any:
+        """Get the data input."""
+
+    @abstractmethod
+    def build_document_transformer(self) -> BaseDocumentTransformer:
+        """Build the text splitter."""

lfx/base/embeddings/aiml_embeddings.py

@@ -0,0 +1,62 @@
+import concurrent.futures
+import json
+
+import httpx
+from pydantic import BaseModel, SecretStr
+
+from lfx.field_typing import Embeddings
+from lfx.log.logger import logger
+
+
+class AIMLEmbeddingsImpl(BaseModel, Embeddings):
+    embeddings_completion_url: str = "https://api.aimlapi.com/v1/embeddings"
+
+    api_key: SecretStr
+    model: str
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        embeddings = [None] * len(texts)
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key.get_secret_value()}",
+        }
+
+        with httpx.Client() as client, concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = []
+            for i, text in enumerate(texts):
+                futures.append((i, executor.submit(self._embed_text, client, headers, text)))
+
+            for index, future in futures:
+                try:
+                    result_data = future.result()
+                    if len(result_data["data"]) != 1:
+                        msg = f"Expected one embedding, got {len(result_data['data'])}"
+                        raise ValueError(msg)
+                    embeddings[index] = result_data["data"][0]["embedding"]
+                except (
+                    httpx.HTTPStatusError,
+                    httpx.RequestError,
+                    json.JSONDecodeError,
+                    KeyError,
+                    ValueError,
+                ):
+                    logger.exception("Error occurred")
+                    raise
+
+        return embeddings  # type: ignore[return-value]
+
+    def _embed_text(self, client: httpx.Client, headers: dict, text: str) -> dict:
+        payload = {
+            "model": self.model,
+            "input": text,
+        }
+        response = client.post(
+            self.embeddings_completion_url,
+            headers=headers,
+            json=payload,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def embed_query(self, text: str) -> list[float]:
+        return self.embed_documents([text])[0]

lfx/base/embeddings/model.py

@@ -0,0 +1,26 @@
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing import Embeddings
+from lfx.io import Output
+
+
+class LCEmbeddingsModel(Component):
+    trace_type = "embedding"
+
+    outputs = [
+        Output(display_name="Embedding Model", name="embeddings", method="build_embeddings"),
+    ]
+
+    def _validate_outputs(self) -> None:
+        required_output_methods = ["build_embeddings"]
+        output_names = [output.name for output in self.outputs]
+        for method_name in required_output_methods:
+            if method_name not in output_names:
+                msg = f"Output with name '{method_name}' must be defined."
+                raise ValueError(msg)
+            if not hasattr(self, method_name):
+                msg = f"Method '{method_name}' must be defined."
+                raise ValueError(msg)
+
+    def build_embeddings(self) -> Embeddings:
+        msg = "You must implement the build_embeddings method in your class."
+        raise NotImplementedError(msg)

lfx/base/flow_processing/utils.py

@@ -0,0 +1,86 @@
+from lfx.graph.schema import ResultData, RunOutputs
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.schema.message import Message
+
+
+def build_data_from_run_outputs(run_outputs: RunOutputs) -> list[Data]:
+    """Build a list of data from the given RunOutputs.
+
+    Args:
+        run_outputs (RunOutputs): The RunOutputs object containing the output data.
+
+    Returns:
+        List[Data]: A list of data built from the RunOutputs.
+
+    """
+    if not run_outputs:
+        return []
+    data = []
+    for result_data in run_outputs.outputs:
+        if result_data:
+            data.extend(build_data_from_result_data(result_data))
+    return data
+
+
+def build_data_from_result_data(result_data: ResultData) -> list[Data]:
+    """Build a list of data from the given ResultData.
+
+    Args:
+        result_data (ResultData): The ResultData object containing the result data.
+
+    Returns:
+        List[Data]: A list of data built from the ResultData.
+
+    """
+    messages = result_data.messages
+
+    if not messages:
+        return []
+    data = []
+
+    # Handle results without chat messages (calling flow)
+    if not messages:
+        # Result with a single record
+        if isinstance(result_data.artifacts, dict):
+            data.append(Data(data=result_data.artifacts))
+        # List of artifacts
+        elif isinstance(result_data.artifacts, list):
+            for artifact in result_data.artifacts:
+                # If multiple records are found as artifacts, return as-is
+                if isinstance(artifact, Data):
+                    data.append(artifact)
+                else:
+                    # Warn about unknown output type
+                    logger.warning(f"Unable to build record output from unknown ResultData.artifact: {artifact}")
+        # Chat or text output
+        elif result_data.results:
+            data.append(Data(data={"result": result_data.results}, text_key="result"))
+            return data
+        else:
+            return []
+
+    if isinstance(result_data.results, dict):
+        for name, result in result_data.results.items():
+            dataobj: Data | Message | None
+            dataobj = result if isinstance(result, Message) else Data(data=result, text_key=name)
+
+            data.append(dataobj)
+    else:
+        data.append(Data(data=result_data.results))
+    return data
+
+
+def format_flow_output_data(data: list[Data]) -> str:
+    """Format the flow output data into a string.
+
+    Args:
+        data (List[Data]): The list of data to format.
+
+    Returns:
+        str: The formatted flow output data.
+
+    """
+    result = "Flow run output:\n"
+    results = "\n".join([value.get_text() if hasattr(value, "get_text") else str(value) for value in data])
+    return result + results