langflow-base-nightly 0.5.0.dev39__py3-none-any.whl → 0.5.1.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langflow/api/router.py +2 -0
- langflow/api/v1/__init__.py +2 -0
- langflow/api/v1/endpoints.py +7 -1
- langflow/api/v1/openai_responses.py +545 -0
- langflow/components/data/file.py +302 -376
- langflow/components/docling/docling_inline.py +56 -4
- langflow/components/nvidia/nvidia_ingest.py +3 -2
- langflow/components/youtube/channel.py +1 -1
- langflow/custom/custom_component/custom_component.py +11 -0
- langflow/graph/graph/base.py +3 -1
- langflow/initial_setup/starter_projects/Basic Prompt Chaining.json +1 -1
- langflow/initial_setup/starter_projects/Basic Prompting.json +1 -1
- langflow/initial_setup/starter_projects/Blog Writer.json +2 -2
- langflow/initial_setup/starter_projects/Custom Component Generator.json +1 -1
- langflow/initial_setup/starter_projects/Document Q&A.json +2 -2
- langflow/initial_setup/starter_projects/Financial Report Parser.json +1 -1
- langflow/initial_setup/starter_projects/Hybrid Search RAG.json +2 -2
- langflow/initial_setup/starter_projects/Image Sentiment Analysis.json +1 -1
- langflow/initial_setup/starter_projects/Instagram Copywriter.json +2 -2
- langflow/initial_setup/starter_projects/Invoice Summarizer.json +1 -1
- langflow/initial_setup/starter_projects/Knowledge Ingestion.json +2 -2
- langflow/initial_setup/starter_projects/Knowledge Retrieval.json +1 -1
- langflow/initial_setup/starter_projects/Market Research.json +2 -2
- langflow/initial_setup/starter_projects/Meeting Summary.json +3 -3
- langflow/initial_setup/starter_projects/Memory Chatbot.json +1 -1
- langflow/initial_setup/starter_projects/News Aggregator.json +3 -3
- langflow/initial_setup/starter_projects/Nvidia Remix.json +2 -2
- langflow/initial_setup/starter_projects/Pok/303/251dex Agent.json" +2 -2
- langflow/initial_setup/starter_projects/Portfolio Website Code Generator.json +2 -2
- langflow/initial_setup/starter_projects/Price Deal Finder.json +3 -3
- langflow/initial_setup/starter_projects/Research Agent.json +2 -2
- langflow/initial_setup/starter_projects/Research Translation Loop.json +1 -1
- langflow/initial_setup/starter_projects/SEO Keyword Generator.json +1 -1
- langflow/initial_setup/starter_projects/SaaS Pricing.json +1 -1
- langflow/initial_setup/starter_projects/Search agent.json +2 -2
- langflow/initial_setup/starter_projects/Sequential Tasks Agents.json +3 -3
- langflow/initial_setup/starter_projects/Simple Agent.json +2 -2
- langflow/initial_setup/starter_projects/Social Media Agent.json +5 -5
- langflow/initial_setup/starter_projects/Text Sentiment Analysis.json +3 -3
- langflow/initial_setup/starter_projects/Travel Planning Agents.json +1 -1
- langflow/initial_setup/starter_projects/Twitter Thread Generator.json +1 -1
- langflow/initial_setup/starter_projects/Vector Store RAG.json +5 -5
- langflow/initial_setup/starter_projects/Youtube Analysis.json +2 -2
- langflow/schema/openai_responses_schemas.py +74 -0
- {langflow_base_nightly-0.5.0.dev39.dist-info → langflow_base_nightly-0.5.1.dev1.dist-info}/METADATA +1 -1
- {langflow_base_nightly-0.5.0.dev39.dist-info → langflow_base_nightly-0.5.1.dev1.dist-info}/RECORD +48 -46
- {langflow_base_nightly-0.5.0.dev39.dist-info → langflow_base_nightly-0.5.1.dev1.dist-info}/WHEEL +0 -0
- {langflow_base_nightly-0.5.0.dev39.dist-info → langflow_base_nightly-0.5.1.dev1.dist-info}/entry_points.txt +0 -0
|
@@ -61,11 +61,11 @@ class DoclingInlineComponent(BaseFileComponent):
|
|
|
61
61
|
),
|
|
62
62
|
DropdownInput(
|
|
63
63
|
name="ocr_engine",
|
|
64
|
-
display_name="
|
|
65
|
-
info="OCR engine to use",
|
|
66
|
-
options=["", "easyocr", "tesserocr", "rapidocr", "ocrmac"],
|
|
64
|
+
display_name="OCR Engine",
|
|
65
|
+
info="OCR engine to use. None will disable OCR.",
|
|
66
|
+
options=["None", "easyocr", "tesserocr", "rapidocr", "ocrmac"],
|
|
67
67
|
real_time_refresh=False,
|
|
68
|
-
value="",
|
|
68
|
+
value="None",
|
|
69
69
|
),
|
|
70
70
|
# TODO: expose more Docling options
|
|
71
71
|
]
|
|
@@ -130,6 +130,58 @@ class DoclingInlineComponent(BaseFileComponent):
|
|
|
130
130
|
self.log("Warning: Process still alive after SIGKILL")
|
|
131
131
|
|
|
132
132
|
def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:
|
|
133
|
+
try:
|
|
134
|
+
from docling.datamodel.base_models import InputFormat
|
|
135
|
+
from docling.datamodel.pipeline_options import (
|
|
136
|
+
OcrOptions,
|
|
137
|
+
PdfPipelineOptions,
|
|
138
|
+
VlmPipelineOptions,
|
|
139
|
+
)
|
|
140
|
+
from docling.document_converter import DocumentConverter, FormatOption, PdfFormatOption
|
|
141
|
+
from docling.models.factories import get_ocr_factory
|
|
142
|
+
from docling.pipeline.vlm_pipeline import VlmPipeline
|
|
143
|
+
except ImportError as e:
|
|
144
|
+
msg = (
|
|
145
|
+
"Docling is an optional dependency. Install with `uv pip install 'langflow[docling]'` or refer to the "
|
|
146
|
+
"documentation on how to install optional dependencies."
|
|
147
|
+
)
|
|
148
|
+
raise ImportError(msg) from e
|
|
149
|
+
|
|
150
|
+
# Configure the standard PDF pipeline
|
|
151
|
+
def _get_standard_opts() -> PdfPipelineOptions:
|
|
152
|
+
pipeline_options = PdfPipelineOptions()
|
|
153
|
+
pipeline_options.do_ocr = self.ocr_engine != "None"
|
|
154
|
+
if pipeline_options.do_ocr:
|
|
155
|
+
ocr_factory = get_ocr_factory(
|
|
156
|
+
allow_external_plugins=False,
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
ocr_options: OcrOptions = ocr_factory.create_options(
|
|
160
|
+
kind=self.ocr_engine,
|
|
161
|
+
)
|
|
162
|
+
pipeline_options.ocr_options = ocr_options
|
|
163
|
+
return pipeline_options
|
|
164
|
+
|
|
165
|
+
# Configure the VLM pipeline
|
|
166
|
+
def _get_vlm_opts() -> VlmPipelineOptions:
|
|
167
|
+
return VlmPipelineOptions()
|
|
168
|
+
|
|
169
|
+
# Configure the main format options and create the DocumentConverter()
|
|
170
|
+
def _get_converter() -> DocumentConverter:
|
|
171
|
+
if self.pipeline == "standard":
|
|
172
|
+
pdf_format_option = PdfFormatOption(
|
|
173
|
+
pipeline_options=_get_standard_opts(),
|
|
174
|
+
)
|
|
175
|
+
elif self.pipeline == "vlm":
|
|
176
|
+
pdf_format_option = PdfFormatOption(pipeline_cls=VlmPipeline, pipeline_options=_get_vlm_opts())
|
|
177
|
+
|
|
178
|
+
format_options: dict[InputFormat, FormatOption] = {
|
|
179
|
+
InputFormat.PDF: pdf_format_option,
|
|
180
|
+
InputFormat.IMAGE: pdf_format_option,
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
return DocumentConverter(format_options=format_options)
|
|
184
|
+
|
|
133
185
|
file_paths = [file.path for file in file_list if file.path]
|
|
134
186
|
|
|
135
187
|
if not file_paths:
|
|
@@ -21,8 +21,9 @@ class NvidiaIngestComponent(BaseFileComponent):
|
|
|
21
21
|
VALID_EXTENSIONS = ["pdf", "docx", "pptx", "jpeg", "png", "svg", "tiff", "txt"]
|
|
22
22
|
except ImportError:
|
|
23
23
|
msg = (
|
|
24
|
-
"NVIDIA Retriever Extraction (nv-ingest)
|
|
25
|
-
"
|
|
24
|
+
"NVIDIA Retriever Extraction (nv-ingest) is an optional dependency. "
|
|
25
|
+
"Install with `uv pip install 'langflow[nv-ingest]'` "
|
|
26
|
+
"(requires Python 3.12>=)"
|
|
26
27
|
)
|
|
27
28
|
VALID_EXTENSIONS = [msg]
|
|
28
29
|
|
|
@@ -220,7 +220,7 @@ class YouTubeChannelComponent(Component):
|
|
|
220
220
|
|
|
221
221
|
return DataFrame(channel_df)
|
|
222
222
|
|
|
223
|
-
except (HttpError, HTTPError
|
|
223
|
+
except (HttpError, HTTPError) as e:
|
|
224
224
|
return DataFrame(pd.DataFrame({"error": [str(e)]}))
|
|
225
225
|
finally:
|
|
226
226
|
if youtube:
|
|
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
|
|
8
8
|
import yaml
|
|
9
9
|
from cachetools import TTLCache
|
|
10
10
|
from langchain_core.documents import Document
|
|
11
|
+
from loguru import logger
|
|
11
12
|
from pydantic import BaseModel
|
|
12
13
|
|
|
13
14
|
from langflow.custom.custom_component.base_component import BaseComponent
|
|
@@ -421,6 +422,16 @@ class CustomComponent(BaseComponent):
|
|
|
421
422
|
if hasattr(self, "_user_id") and not self.user_id:
|
|
422
423
|
msg = f"User id is not set for {self.__class__.__name__}"
|
|
423
424
|
raise ValueError(msg)
|
|
425
|
+
|
|
426
|
+
# Check graph context for request-level variable overrides first
|
|
427
|
+
if hasattr(self, "graph") and self.graph and hasattr(self.graph, "context"):
|
|
428
|
+
context = self.graph.context
|
|
429
|
+
if context and "request_variables" in context:
|
|
430
|
+
request_variables = context["request_variables"]
|
|
431
|
+
if name in request_variables:
|
|
432
|
+
logger.debug(f"Found context override for variable '{name}': {request_variables[name]}")
|
|
433
|
+
return request_variables[name]
|
|
434
|
+
|
|
424
435
|
variable_service = get_variable_service() # Get service instance
|
|
425
436
|
# Retrieve and decrypt the variable by name for the current user
|
|
426
437
|
if isinstance(self.user_id, str):
|
langflow/graph/graph/base.py
CHANGED
|
@@ -1047,6 +1047,7 @@ class Graph:
|
|
|
1047
1047
|
flow_id: str | None = None,
|
|
1048
1048
|
flow_name: str | None = None,
|
|
1049
1049
|
user_id: str | None = None,
|
|
1050
|
+
context: dict | None = None,
|
|
1050
1051
|
) -> Graph:
|
|
1051
1052
|
"""Creates a graph from a payload.
|
|
1052
1053
|
|
|
@@ -1055,6 +1056,7 @@ class Graph:
|
|
|
1055
1056
|
flow_id: The ID of the flow.
|
|
1056
1057
|
flow_name: The flow name.
|
|
1057
1058
|
user_id: The user ID.
|
|
1059
|
+
context: Optional context dictionary for request-specific data.
|
|
1058
1060
|
|
|
1059
1061
|
Returns:
|
|
1060
1062
|
Graph: The created graph.
|
|
@@ -1064,7 +1066,7 @@ class Graph:
|
|
|
1064
1066
|
try:
|
|
1065
1067
|
vertices = payload["nodes"]
|
|
1066
1068
|
edges = payload["edges"]
|
|
1067
|
-
graph = cls(flow_id=flow_id, flow_name=flow_name, user_id=user_id)
|
|
1069
|
+
graph = cls(flow_id=flow_id, flow_name=flow_name, user_id=user_id, context=context)
|
|
1068
1070
|
graph.add_nodes_and_edges(vertices, edges)
|
|
1069
1071
|
except KeyError as exc:
|
|
1070
1072
|
logger.exception(exc)
|
|
@@ -486,7 +486,7 @@
|
|
|
486
486
|
},
|
|
487
487
|
{
|
|
488
488
|
"name": "fastapi",
|
|
489
|
-
"version": "0.
|
|
489
|
+
"version": "0.116.1"
|
|
490
490
|
},
|
|
491
491
|
{
|
|
492
492
|
"name": "langflow",
|
|
@@ -1018,7 +1018,7 @@
|
|
|
1018
1018
|
"dependencies": [
|
|
1019
1019
|
{
|
|
1020
1020
|
"name": "requests",
|
|
1021
|
-
"version": "2.32.
|
|
1021
|
+
"version": "2.32.5"
|
|
1022
1022
|
},
|
|
1023
1023
|
{
|
|
1024
1024
|
"name": "bs4",
|
|
@@ -460,7 +460,7 @@
|
|
|
460
460
|
},
|
|
461
461
|
{
|
|
462
462
|
"name": "fastapi",
|
|
463
|
-
"version": "0.
|
|
463
|
+
"version": "0.116.1"
|
|
464
464
|
},
|
|
465
465
|
{
|
|
466
466
|
"name": "langflow",
|
|
@@ -1302,7 +1302,7 @@
|
|
|
1302
1302
|
"show": true,
|
|
1303
1303
|
"title_case": false,
|
|
1304
1304
|
"type": "code",
|
|
1305
|
-
"value": "\"\"\"Enhanced file component v2 with mypy and ruff compliance.\"\"\"\n\nfrom __future__ import annotations\n\nfrom copy import deepcopy\nfrom enum import Enum\nfrom typing import TYPE_CHECKING, Any\n\nfrom langflow.base.data.base_file import BaseFileComponent\nfrom langflow.base.data.utils import TEXT_FILE_TYPES, parallel_load_data, parse_text_file_to_data\nfrom langflow.io import (\n BoolInput,\n DropdownInput,\n FileInput,\n IntInput,\n MessageTextInput,\n Output,\n StrInput,\n)\nfrom langflow.schema.data import Data\nfrom langflow.schema.message import Message\n\nif TYPE_CHECKING:\n from langflow.schema import DataFrame\n\n\nclass MockConversionStatus(Enum):\n \"\"\"Mock ConversionStatus for fallback compatibility.\"\"\"\n\n SUCCESS = \"success\"\n FAILURE = \"failure\"\n\n\nclass MockInputFormat(Enum):\n \"\"\"Mock InputFormat for fallback compatibility.\"\"\"\n\n PDF = \"pdf\"\n IMAGE = \"image\"\n\n\nclass MockImageRefMode(Enum):\n \"\"\"Mock ImageRefMode for fallback compatibility.\"\"\"\n\n PLACEHOLDER = \"placeholder\"\n EMBEDDED = \"embedded\"\n\n\nclass DoclingImports:\n \"\"\"Container for docling imports with type information.\"\"\"\n\n def __init__(\n self,\n conversion_status: type[Enum],\n input_format: type[Enum],\n document_converter: type,\n image_ref_mode: type[Enum],\n strategy: str,\n ) -> None:\n self.conversion_status = conversion_status\n self.input_format = input_format\n self.document_converter = document_converter\n self.image_ref_mode = image_ref_mode\n self.strategy = strategy\n\n\nclass FileComponent(BaseFileComponent):\n \"\"\"Enhanced file component v2 that combines standard file loading with optional Docling processing and export.\n\n This component supports all features of the standard File component, plus an advanced mode\n that enables Docling document processing and export to various formats (Markdown, HTML, etc.).\n \"\"\"\n\n display_name = \"File\"\n description = \"Loads content from files with optional advanced document processing and export using Docling.\"\n documentation: str = \"https://docs.langflow.org/components-data#file\"\n icon = \"file-text\"\n name = \"File\"\n\n # Docling supported formats from original component\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n *TEXT_FILE_TYPES,\n ]\n\n # Fixed export settings\n EXPORT_FORMAT = \"Markdown\"\n IMAGE_MODE = \"placeholder\"\n\n _base_inputs = deepcopy(BaseFileComponent._base_inputs)\n\n for input_item in _base_inputs:\n if isinstance(input_item, FileInput) and input_item.name == \"path\":\n input_item.real_time_refresh = True\n break\n\n inputs = [\n *_base_inputs,\n BoolInput(\n name=\"advanced_mode\",\n display_name=\"Advanced Parser\",\n value=False,\n real_time_refresh=True,\n info=(\n \"Enable advanced document processing and export with Docling for PDFs, images, and office documents. \"\n \"Available only for single file processing.\"\n ),\n show=False,\n ),\n DropdownInput(\n name=\"pipeline\",\n display_name=\"Pipeline\",\n info=\"Docling pipeline to use\",\n options=[\"standard\", \"vlm\"],\n value=\"standard\",\n advanced=True,\n ),\n DropdownInput(\n name=\"ocr_engine\",\n display_name=\"OCR Engine\",\n info=\"OCR engine to use. Only available when pipeline is set to 'standard'.\",\n options=[\"\", \"easyocr\"],\n value=\"\",\n show=False,\n advanced=True,\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"<!-- image -->\",\n advanced=True,\n show=False,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder between pages in the markdown output.\",\n value=\"\",\n advanced=True,\n show=False,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n show=False,\n ),\n BoolInput(\n name=\"use_multithreading\",\n display_name=\"[Deprecated] Use Multithreading\",\n advanced=True,\n value=True,\n info=\"Set 'Processing Concurrency' greater than 1 to enable multithreading.\",\n ),\n IntInput(\n name=\"concurrency_multithreading\",\n display_name=\"Processing Concurrency\",\n advanced=True,\n info=\"When multiple files are being processed, the number of files to process concurrently.\",\n value=1,\n ),\n BoolInput(\n name=\"markdown\",\n display_name=\"Markdown Export\",\n info=\"Export processed documents to Markdown format. Only available when advanced mode is enabled.\",\n value=False,\n show=False,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n ]\n\n def _path_value(self, template) -> list[str]:\n # Get current path value\n return template.get(\"path\", {}).get(\"file_path\", [])\n\n def update_build_config(\n self,\n build_config: dict[str, Any],\n field_value: Any,\n field_name: str | None = None,\n ) -> dict[str, Any]:\n \"\"\"Update build configuration to show/hide fields based on file count and advanced_mode.\"\"\"\n if field_name == \"path\":\n # Get current path value\n path_value = self._path_value(build_config)\n file_path = path_value[0] if len(path_value) > 0 else \"\"\n\n # Show/hide Advanced Parser based on file count (only for single files)\n file_count = len(field_value) if field_value else 0\n if file_count == 1 and not file_path.endswith((\".csv\", \".xlsx\", \".parquet\")):\n build_config[\"advanced_mode\"][\"show\"] = True\n else:\n build_config[\"advanced_mode\"][\"show\"] = False\n build_config[\"advanced_mode\"][\"value\"] = False # Reset to False when hidden\n\n # Hide all advanced fields when Advanced Parser is not available\n advanced_fields = [\n \"pipeline\",\n \"ocr_engine\",\n \"doc_key\",\n \"md_image_placeholder\",\n \"md_page_break_placeholder\",\n ]\n for field in advanced_fields:\n if field in build_config:\n build_config[field][\"show\"] = False\n\n elif field_name == \"advanced_mode\":\n # Show/hide advanced fields based on advanced_mode (only if single file)\n advanced_fields = [\n \"pipeline\",\n \"ocr_engine\",\n \"doc_key\",\n \"md_image_placeholder\",\n \"md_page_break_placeholder\",\n ]\n\n for field in advanced_fields:\n if field in build_config:\n build_config[field][\"show\"] = field_value\n\n return build_config\n\n def update_outputs(self, frontend_node: dict[str, Any], field_name: str, field_value: Any) -> dict[str, Any]: # noqa: ARG002\n \"\"\"Dynamically show outputs based on the number of files and their types.\"\"\"\n if field_name not in [\"path\", \"advanced_mode\"]:\n return frontend_node\n\n # Add outputs based on the number of files in the path\n template = frontend_node.get(\"template\", {})\n path_value = self._path_value(template)\n if len(path_value) == 0:\n return frontend_node\n\n # Clear existing outputs\n frontend_node[\"outputs\"] = []\n\n if len(path_value) == 1:\n # We need to check if the file is structured content\n file_path = path_value[0] if field_name == \"path\" else frontend_node[\"template\"][\"path\"][\"file_path\"][0]\n if file_path.endswith((\".csv\", \".xlsx\", \".parquet\")):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"dataframe\", method=\"load_files_structured\"),\n )\n elif file_path.endswith(\".json\"):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"json\", method=\"load_files_json\"),\n )\n\n # Add outputs based on advanced mode\n advanced_mode = frontend_node.get(\"template\", {}).get(\"advanced_mode\", {}).get(\"value\", False)\n\n if advanced_mode:\n # Advanced mode: Structured Output, Markdown, and File Path\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Output\", name=\"advanced\", method=\"load_files_advanced\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Markdown\", name=\"markdown\", method=\"load_files_markdown\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n # Normal mode: Raw Content and File Path\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n # For multiple files, we show the files output (DataFrame format)\n # Advanced Parser is not available for multiple files\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Files\", name=\"dataframe\", method=\"load_files\"),\n )\n\n return frontend_node\n\n def _try_import_docling(self) -> DoclingImports | None:\n \"\"\"Try different import strategies for docling components.\"\"\"\n # Try strategy 1: Latest docling structure\n try:\n from docling.datamodel.base_models import ConversionStatus, InputFormat # type: ignore[import-untyped]\n from docling.document_converter import DocumentConverter # type: ignore[import-untyped]\n from docling_core.types.doc import ImageRefMode # type: ignore[import-untyped]\n\n self.log(\"Using latest docling import structure\")\n return DoclingImports(\n conversion_status=ConversionStatus,\n input_format=InputFormat,\n document_converter=DocumentConverter,\n image_ref_mode=ImageRefMode,\n strategy=\"latest\",\n )\n except ImportError as e:\n self.log(f\"Latest docling structure failed: {e}\")\n\n # Try strategy 2: Alternative import paths\n try:\n from docling.document_converter import DocumentConverter # type: ignore[import-untyped]\n from docling_core.types.doc import ImageRefMode # type: ignore[import-untyped]\n\n # Try to get ConversionStatus from different locations\n conversion_status: type[Enum] = MockConversionStatus\n input_format: type[Enum] = MockInputFormat\n\n try:\n from docling_core.types import ConversionStatus, InputFormat # type: ignore[import-untyped]\n\n conversion_status = ConversionStatus\n input_format = InputFormat\n except ImportError:\n try:\n from docling.datamodel import ConversionStatus, InputFormat # type: ignore[import-untyped]\n\n conversion_status = ConversionStatus\n input_format = InputFormat\n except ImportError:\n # Use mock enums if we can't find them\n pass\n\n self.log(\"Using alternative docling import structure\")\n return DoclingImports(\n conversion_status=conversion_status,\n input_format=input_format,\n document_converter=DocumentConverter,\n image_ref_mode=ImageRefMode,\n strategy=\"alternative\",\n )\n except ImportError as e:\n self.log(f\"Alternative docling structure failed: {e}\")\n\n # Try strategy 3: Basic converter only\n try:\n from docling.document_converter import DocumentConverter # type: ignore[import-untyped]\n\n self.log(\"Using basic docling import structure with mocks\")\n return DoclingImports(\n conversion_status=MockConversionStatus,\n input_format=MockInputFormat,\n document_converter=DocumentConverter,\n image_ref_mode=MockImageRefMode,\n strategy=\"basic\",\n )\n except ImportError as e:\n self.log(f\"Basic docling structure failed: {e}\")\n\n # Strategy 4: Complete fallback - return None to indicate failure\n return None\n\n def _create_advanced_converter(self, docling_imports: DoclingImports) -> Any:\n \"\"\"Create advanced converter with pipeline options if available.\"\"\"\n try:\n from docling.datamodel.pipeline_options import PdfPipelineOptions # type: ignore[import-untyped]\n from docling.document_converter import PdfFormatOption # type: ignore[import-untyped]\n\n document_converter = docling_imports.document_converter\n input_format = docling_imports.input_format\n\n # Create basic pipeline options\n pipeline_options = PdfPipelineOptions()\n\n # Configure OCR if specified and available\n if self.ocr_engine:\n try:\n from docling.models.factories import get_ocr_factory # type: ignore[import-untyped]\n\n pipeline_options.do_ocr = True\n ocr_factory = get_ocr_factory(allow_external_plugins=False)\n ocr_options = ocr_factory.create_options(kind=self.ocr_engine)\n pipeline_options.ocr_options = ocr_options\n self.log(f\"Configured OCR with engine: {self.ocr_engine}\")\n except Exception as e: # noqa: BLE001\n self.log(f\"Could not configure OCR: {e}, proceeding without OCR\")\n pipeline_options.do_ocr = False\n\n # Create format options\n pdf_format_option = PdfFormatOption(pipeline_options=pipeline_options)\n format_options = {}\n if hasattr(input_format, \"PDF\"):\n format_options[input_format.PDF] = pdf_format_option\n if hasattr(input_format, \"IMAGE\"):\n format_options[input_format.IMAGE] = pdf_format_option\n\n return document_converter(format_options=format_options)\n\n except Exception as e: # noqa: BLE001\n self.log(f\"Could not create advanced converter: {e}, using basic converter\")\n return docling_imports.document_converter()\n\n def _is_docling_compatible(self, file_path: str) -> bool:\n \"\"\"Check if file is compatible with Docling processing.\"\"\"\n # All VALID_EXTENSIONS are Docling compatible (except for TEXT_FILE_TYPES which may overlap)\n docling_extensions = [\n \".adoc\",\n \".asciidoc\",\n \".asc\",\n \".bmp\",\n \".csv\",\n \".dotx\",\n \".dotm\",\n \".docm\",\n \".docx\",\n \".htm\",\n \".html\",\n \".jpeg\",\n \".json\",\n \".md\",\n \".pdf\",\n \".png\",\n \".potx\",\n \".ppsx\",\n \".pptm\",\n \".potm\",\n \".ppsm\",\n \".pptx\",\n \".tiff\",\n \".txt\",\n \".xls\",\n \".xlsx\",\n \".xhtml\",\n \".xml\",\n \".webp\",\n ]\n return any(file_path.lower().endswith(ext) for ext in docling_extensions)\n\n def process_files(\n self,\n file_list: list[BaseFileComponent.BaseFile],\n ) -> list[BaseFileComponent.BaseFile]:\n \"\"\"Process files using standard parsing or Docling based on advanced_mode and file type.\"\"\"\n\n def process_file_standard(file_path: str, *, silent_errors: bool = False) -> Data | None:\n \"\"\"Process a single file using standard text parsing.\"\"\"\n try:\n return parse_text_file_to_data(file_path, silent_errors=silent_errors)\n except FileNotFoundError as e:\n msg = f\"File not found: {file_path}. Error: {e}\"\n self.log(msg)\n if not silent_errors:\n raise\n return None\n except Exception as e:\n msg = f\"Unexpected error processing {file_path}: {e}\"\n self.log(msg)\n if not silent_errors:\n raise\n return None\n\n def process_file_docling(file_path: str, *, silent_errors: bool = False) -> Data | None:\n \"\"\"Process a single file using Docling if compatible, otherwise standard processing.\"\"\"\n # Try Docling first if file is compatible and advanced mode is enabled\n try:\n return self._process_with_docling_and_export(file_path)\n except Exception as e: # noqa: BLE001\n self.log(f\"Docling processing failed for {file_path}: {e}, falling back to standard processing\")\n if not silent_errors:\n # Return error data instead of raising\n return Data(data={\"error\": f\"Docling processing failed: {e}\", \"file_path\": file_path})\n\n return None\n\n if not file_list:\n msg = \"No files to process.\"\n raise ValueError(msg)\n\n file_path = str(file_list[0].path)\n if self.advanced_mode and self._is_docling_compatible(file_path):\n processed_data = process_file_docling(file_path)\n if not processed_data:\n msg = f\"Failed to process file with Docling: {file_path}\"\n raise ValueError(msg)\n\n # Serialize processed data to match Data structure\n serialized_data = processed_data.serialize_model()\n\n # Now, if doc is nested, we need to unravel it\n clean_data: list[Data | None] = [processed_data]\n\n # This is where we've manually processed the data\n try:\n if \"exported_content\" not in serialized_data:\n clean_data = [\n Data(\n data={\n \"file_path\": file_path,\n **(\n item[\"element\"]\n if \"element\" in item\n else {k: v for k, v in item.items() if k != \"file_path\"}\n ),\n }\n )\n for item in serialized_data[\"doc\"]\n ]\n except Exception as _: # noqa: BLE001\n raise ValueError(serialized_data) from None\n\n # Repeat file_list to match the number of processed data elements\n final_data: list[Data | None] = clean_data\n return self.rollup_data(file_list, final_data)\n\n concurrency = 1 if not self.use_multithreading else max(1, self.concurrency_multithreading)\n file_count = len(file_list)\n\n self.log(f\"Starting parallel processing of {file_count} files with concurrency: {concurrency}.\")\n file_paths = [str(file.path) for file in file_list]\n my_data = parallel_load_data(\n file_paths,\n silent_errors=self.silent_errors,\n load_function=process_file_standard,\n max_concurrency=concurrency,\n )\n\n return self.rollup_data(file_list, my_data)\n\n def load_files_advanced(self) -> DataFrame:\n \"\"\"Load files using advanced Docling processing and export to an advanced format.\"\"\"\n # TODO: Update\n self.markdown = False\n return self.load_files()\n\n def load_files_markdown(self) -> Message:\n \"\"\"Load files using advanced Docling processing and export to Markdown format.\"\"\"\n self.markdown = True\n result = self.load_files()\n return Message(text=str(result.text[0]))\n\n def _process_with_docling_and_export(self, file_path: str) -> Data:\n \"\"\"Process a single file with Docling and export to the specified format.\"\"\"\n # Import docling components only when needed\n docling_imports = self._try_import_docling()\n\n if docling_imports is None:\n msg = \"Docling not available for advanced processing\"\n raise ImportError(msg)\n\n conversion_status = docling_imports.conversion_status\n document_converter = docling_imports.document_converter\n image_ref_mode = docling_imports.image_ref_mode\n\n try:\n # Create converter based on strategy and pipeline setting\n if docling_imports.strategy == \"latest\" and self.pipeline == \"standard\":\n converter = self._create_advanced_converter(docling_imports)\n else:\n # Use basic converter for compatibility\n converter = document_converter()\n self.log(\"Using basic DocumentConverter for Docling processing\")\n\n # Process single file\n result = converter.convert(file_path)\n\n # Check if conversion was successful\n success = False\n if hasattr(result, \"status\"):\n if hasattr(conversion_status, \"SUCCESS\"):\n success = result.status == conversion_status.SUCCESS\n else:\n success = str(result.status).lower() == \"success\"\n elif hasattr(result, \"document\"):\n # If no status but has document, assume success\n success = result.document is not None\n\n if not success:\n return Data(data={\"error\": \"Docling conversion failed\", \"file_path\": file_path})\n\n if self.markdown:\n self.log(\"Exporting document to Markdown format\")\n # Export the document to the specified format\n exported_content = self._export_document(result.document, image_ref_mode)\n\n return Data(\n text=exported_content,\n data={\n \"exported_content\": exported_content,\n \"export_format\": self.EXPORT_FORMAT,\n \"file_path\": file_path,\n },\n )\n\n return Data(\n data={\n \"doc\": self.docling_to_dataframe_simple(result.document.export_to_dict()),\n \"export_format\": self.EXPORT_FORMAT,\n \"file_path\": file_path,\n }\n )\n\n except Exception as e: # noqa: BLE001\n return Data(data={\"error\": f\"Docling processing error: {e!s}\", \"file_path\": file_path})\n\n def docling_to_dataframe_simple(self, doc):\n \"\"\"Extract all text elements into a simple DataFrame.\"\"\"\n return [\n {\n \"page_no\": text[\"prov\"][0][\"page_no\"] if text[\"prov\"] else None,\n \"label\": text[\"label\"],\n \"text\": text[\"text\"],\n \"level\": text.get(\"level\", None), # for headers\n }\n for text in doc[\"texts\"]\n ]\n\n def _export_document(self, document: Any, image_ref_mode: type[Enum]) -> str:\n \"\"\"Export document to Markdown format with placeholder images.\"\"\"\n try:\n image_mode = (\n image_ref_mode(self.IMAGE_MODE) if hasattr(image_ref_mode, self.IMAGE_MODE) else self.IMAGE_MODE\n )\n\n # Always export to Markdown since it's fixed\n return document.export_to_markdown(\n image_mode=image_mode,\n image_placeholder=self.md_image_placeholder,\n page_break_placeholder=self.md_page_break_placeholder,\n )\n\n except Exception as e: # noqa: BLE001\n self.log(f\"Markdown export failed: {e}, using basic text export\")\n # Fallback to basic text export\n try:\n return document.export_to_text()\n except Exception: # noqa: BLE001\n return str(document)\n"
|
|
1305
|
+
"value": "\"\"\"Enhanced file component with clearer structure and Docling isolation.\n\nNotes:\n-----\n- Functionality is preserved with minimal behavioral changes.\n- ALL Docling parsing/export runs in a separate OS process to prevent memory\n growth and native library state from impacting the main Langflow process.\n- Standard text/structured parsing continues to use existing BaseFileComponent\n utilities (and optional threading via `parallel_load_data`).\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport subprocess\nimport sys\nimport textwrap\nfrom copy import deepcopy\nfrom typing import TYPE_CHECKING, Any\n\nfrom langflow.base.data.base_file import BaseFileComponent\nfrom langflow.base.data.utils import TEXT_FILE_TYPES, parallel_load_data, parse_text_file_to_data\nfrom langflow.io import (\n BoolInput,\n DropdownInput,\n FileInput,\n IntInput,\n MessageTextInput,\n Output,\n StrInput,\n)\nfrom langflow.schema.data import Data\nfrom langflow.schema.message import Message\n\nif TYPE_CHECKING:\n from langflow.schema import DataFrame\n\n\nclass FileComponent(BaseFileComponent):\n \"\"\"File component with optional Docling processing (isolated in a subprocess).\"\"\"\n\n display_name = \"File\"\n description = \"Loads content from files with optional advanced document processing and export using Docling.\"\n documentation: str = \"https://docs.langflow.org/components-data#file\"\n icon = \"file-text\"\n name = \"File\"\n\n # Docling-supported/compatible extensions; TEXT_FILE_TYPES are supported by the base loader.\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n *TEXT_FILE_TYPES,\n ]\n\n # Fixed export settings used when markdown export is requested.\n EXPORT_FORMAT = \"Markdown\"\n IMAGE_MODE = \"placeholder\"\n\n # ---- Inputs / Outputs (kept as close to original as possible) -------------------\n _base_inputs = deepcopy(BaseFileComponent._base_inputs)\n for input_item in _base_inputs:\n if isinstance(input_item, FileInput) and input_item.name == \"path\":\n input_item.real_time_refresh = True\n break\n\n inputs = [\n *_base_inputs,\n BoolInput(\n name=\"advanced_mode\",\n display_name=\"Advanced Parser\",\n value=False,\n real_time_refresh=True,\n info=(\n \"Enable advanced document processing and export with Docling for PDFs, images, and office documents. \"\n \"Available only for single file processing.\"\n ),\n show=False,\n ),\n DropdownInput(\n name=\"pipeline\",\n display_name=\"Pipeline\",\n info=\"Docling pipeline to use\",\n options=[\"standard\", \"vlm\"],\n value=\"standard\",\n advanced=True,\n ),\n DropdownInput(\n name=\"ocr_engine\",\n display_name=\"OCR Engine\",\n info=\"OCR engine to use. Only available when pipeline is set to 'standard'.\",\n options=[\"\", \"easyocr\"],\n value=\"\",\n show=False,\n advanced=True,\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"<!-- image -->\",\n advanced=True,\n show=False,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder between pages in the markdown output.\",\n value=\"\",\n advanced=True,\n show=False,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n show=False,\n ),\n # Deprecated input retained for backward-compatibility.\n BoolInput(\n name=\"use_multithreading\",\n display_name=\"[Deprecated] Use Multithreading\",\n advanced=True,\n value=True,\n info=\"Set 'Processing Concurrency' greater than 1 to enable multithreading.\",\n ),\n IntInput(\n name=\"concurrency_multithreading\",\n display_name=\"Processing Concurrency\",\n advanced=True,\n info=\"When multiple files are being processed, the number of files to process concurrently.\",\n value=1,\n ),\n BoolInput(\n name=\"markdown\",\n display_name=\"Markdown Export\",\n info=\"Export processed documents to Markdown format. Only available when advanced mode is enabled.\",\n value=False,\n show=False,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n ]\n\n # ------------------------------ UI helpers --------------------------------------\n\n def _path_value(self, template: dict) -> list[str]:\n \"\"\"Return the list of currently selected file paths from the template.\"\"\"\n return template.get(\"path\", {}).get(\"file_path\", [])\n\n def update_build_config(\n self,\n build_config: dict[str, Any],\n field_value: Any,\n field_name: str | None = None,\n ) -> dict[str, Any]:\n \"\"\"Show/hide Advanced Parser and related fields based on selection context.\"\"\"\n if field_name == \"path\":\n paths = self._path_value(build_config)\n file_path = paths[0] if paths else \"\"\n file_count = len(field_value) if field_value else 0\n\n # Advanced mode only for single (non-tabular) file\n allow_advanced = file_count == 1 and not file_path.endswith((\".csv\", \".xlsx\", \".parquet\"))\n build_config[\"advanced_mode\"][\"show\"] = allow_advanced\n if not allow_advanced:\n build_config[\"advanced_mode\"][\"value\"] = False\n for f in (\"pipeline\", \"ocr_engine\", \"doc_key\", \"md_image_placeholder\", \"md_page_break_placeholder\"):\n if f in build_config:\n build_config[f][\"show\"] = False\n\n elif field_name == \"advanced_mode\":\n for f in (\"pipeline\", \"ocr_engine\", \"doc_key\", \"md_image_placeholder\", \"md_page_break_placeholder\"):\n if f in build_config:\n build_config[f][\"show\"] = bool(field_value)\n\n return build_config\n\n def update_outputs(self, frontend_node: dict[str, Any], field_name: str, field_value: Any) -> dict[str, Any]: # noqa: ARG002\n \"\"\"Dynamically show outputs based on file count/type and advanced mode.\"\"\"\n if field_name not in [\"path\", \"advanced_mode\"]:\n return frontend_node\n\n template = frontend_node.get(\"template\", {})\n paths = self._path_value(template)\n if not paths:\n return frontend_node\n\n frontend_node[\"outputs\"] = []\n if len(paths) == 1:\n file_path = paths[0] if field_name == \"path\" else frontend_node[\"template\"][\"path\"][\"file_path\"][0]\n if file_path.endswith((\".csv\", \".xlsx\", \".parquet\")):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"dataframe\", method=\"load_files_structured\"),\n )\n elif file_path.endswith(\".json\"):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"json\", method=\"load_files_json\"),\n )\n\n advanced_mode = frontend_node.get(\"template\", {}).get(\"advanced_mode\", {}).get(\"value\", False)\n if advanced_mode:\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Output\", name=\"advanced\", method=\"load_files_advanced\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Markdown\", name=\"markdown\", method=\"load_files_markdown\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n # Multiple files => DataFrame output; advanced parser disabled\n frontend_node[\"outputs\"].append(Output(display_name=\"Files\", name=\"dataframe\", method=\"load_files\"))\n\n return frontend_node\n\n # ------------------------------ Core processing ----------------------------------\n\n def _is_docling_compatible(self, file_path: str) -> bool:\n \"\"\"Lightweight extension gate for Docling-compatible types.\"\"\"\n docling_exts = (\n \".adoc\",\n \".asciidoc\",\n \".asc\",\n \".bmp\",\n \".csv\",\n \".dotx\",\n \".dotm\",\n \".docm\",\n \".docx\",\n \".htm\",\n \".html\",\n \".jpeg\",\n \".json\",\n \".md\",\n \".pdf\",\n \".png\",\n \".potx\",\n \".ppsx\",\n \".pptm\",\n \".potm\",\n \".ppsm\",\n \".pptx\",\n \".tiff\",\n \".txt\",\n \".xls\",\n \".xlsx\",\n \".xhtml\",\n \".xml\",\n \".webp\",\n )\n return file_path.lower().endswith(docling_exts)\n\n def _process_docling_in_subprocess(self, file_path: str) -> Data | None:\n \"\"\"Run Docling in a separate OS process and map the result to a Data object.\n\n We avoid multiprocessing pickling by launching `python -c \"<script>\"` and\n passing JSON config via stdin. The child prints a JSON result to stdout.\n \"\"\"\n if not file_path:\n return None\n\n args: dict[str, Any] = {\n \"file_path\": file_path,\n \"markdown\": bool(self.markdown),\n \"image_mode\": str(self.IMAGE_MODE),\n \"md_image_placeholder\": str(self.md_image_placeholder),\n \"md_page_break_placeholder\": str(self.md_page_break_placeholder),\n \"pipeline\": str(self.pipeline),\n \"ocr_engine\": str(self.ocr_engine) if getattr(self, \"ocr_engine\", \"\") else None,\n }\n\n # The child is a tiny, self-contained script to keep memory/state isolated.\n child_script = textwrap.dedent(\n r\"\"\"\n import json, sys\n\n def try_imports():\n # Strategy 1: latest layout\n try:\n from docling.datamodel.base_models import ConversionStatus, InputFormat # type: ignore\n from docling.document_converter import DocumentConverter # type: ignore\n from docling_core.types.doc import ImageRefMode # type: ignore\n return ConversionStatus, InputFormat, DocumentConverter, ImageRefMode, \"latest\"\n except Exception:\n pass\n # Strategy 2: alternative layout\n try:\n from docling.document_converter import DocumentConverter # type: ignore\n try:\n from docling_core.types import ConversionStatus, InputFormat # type: ignore\n except Exception:\n try:\n from docling.datamodel import ConversionStatus, InputFormat # type: ignore\n except Exception:\n class ConversionStatus: SUCCESS = \"success\"\n class InputFormat:\n PDF=\"pdf\"; IMAGE=\"image\"\n try:\n from docling_core.types.doc import ImageRefMode # type: ignore\n except Exception:\n class ImageRefMode:\n PLACEHOLDER=\"placeholder\"; EMBEDDED=\"embedded\"\n return ConversionStatus, InputFormat, DocumentConverter, ImageRefMode, \"alternative\"\n except Exception:\n pass\n # Strategy 3: basic converter only\n try:\n from docling.document_converter import DocumentConverter # type: ignore\n class ConversionStatus: SUCCESS = \"success\"\n class InputFormat:\n PDF=\"pdf\"; IMAGE=\"image\"\n class ImageRefMode:\n PLACEHOLDER=\"placeholder\"; EMBEDDED=\"embedded\"\n return ConversionStatus, InputFormat, DocumentConverter, ImageRefMode, \"basic\"\n except Exception as e:\n raise ImportError(f\"Docling imports failed: {e}\") from e\n\n def create_converter(strategy, input_format, DocumentConverter, pipeline, ocr_engine):\n if strategy == \"latest\" and pipeline == \"standard\":\n try:\n from docling.datamodel.pipeline_options import PdfPipelineOptions # type: ignore\n from docling.document_converter import PdfFormatOption # type: ignore\n pipe = PdfPipelineOptions()\n if ocr_engine:\n try:\n from docling.models.factories import get_ocr_factory # type: ignore\n pipe.do_ocr = True\n fac = get_ocr_factory(allow_external_plugins=False)\n pipe.ocr_options = fac.create_options(kind=ocr_engine)\n except Exception:\n pipe.do_ocr = False\n fmt = {}\n if hasattr(input_format, \"PDF\"):\n fmt[getattr(input_format, \"PDF\")] = PdfFormatOption(pipeline_options=pipe)\n if hasattr(input_format, \"IMAGE\"):\n fmt[getattr(input_format, \"IMAGE\")] = PdfFormatOption(pipeline_options=pipe)\n return DocumentConverter(format_options=fmt)\n except Exception:\n return DocumentConverter()\n return DocumentConverter()\n\n def export_markdown(document, ImageRefMode, image_mode, img_ph, pg_ph):\n try:\n mode = getattr(ImageRefMode, image_mode.upper(), image_mode)\n return document.export_to_markdown(\n image_mode=mode,\n image_placeholder=img_ph,\n page_break_placeholder=pg_ph,\n )\n except Exception:\n try:\n return document.export_to_text()\n except Exception:\n return str(document)\n\n def to_rows(doc_dict):\n rows = []\n for t in doc_dict.get(\"texts\", []):\n prov = t.get(\"prov\") or []\n page_no = None\n if prov and isinstance(prov, list) and isinstance(prov[0], dict):\n page_no = prov[0].get(\"page_no\")\n rows.append({\n \"page_no\": page_no,\n \"label\": t.get(\"label\"),\n \"text\": t.get(\"text\"),\n \"level\": t.get(\"level\"),\n })\n return rows\n\n def main():\n cfg = json.loads(sys.stdin.read())\n file_path = cfg[\"file_path\"]\n markdown = cfg[\"markdown\"]\n image_mode = cfg[\"image_mode\"]\n img_ph = cfg[\"md_image_placeholder\"]\n pg_ph = cfg[\"md_page_break_placeholder\"]\n pipeline = cfg[\"pipeline\"]\n ocr_engine = cfg.get(\"ocr_engine\")\n meta = {\"file_path\": file_path}\n\n try:\n ConversionStatus, InputFormat, DocumentConverter, ImageRefMode, strategy = try_imports()\n converter = create_converter(strategy, InputFormat, DocumentConverter, pipeline, ocr_engine)\n try:\n res = converter.convert(file_path)\n except Exception as e:\n print(json.dumps({\"ok\": False, \"error\": f\"Docling conversion error: {e}\", \"meta\": meta}))\n return\n\n ok = False\n if hasattr(res, \"status\"):\n try:\n ok = (res.status == ConversionStatus.SUCCESS) or (str(res.status).lower() == \"success\")\n except Exception:\n ok = (str(res.status).lower() == \"success\")\n if not ok and hasattr(res, \"document\"):\n ok = getattr(res, \"document\", None) is not None\n if not ok:\n print(json.dumps({\"ok\": False, \"error\": \"Docling conversion failed\", \"meta\": meta}))\n return\n\n doc = getattr(res, \"document\", None)\n if doc is None:\n print(json.dumps({\"ok\": False, \"error\": \"Docling produced no document\", \"meta\": meta}))\n return\n\n if markdown:\n text = export_markdown(doc, ImageRefMode, image_mode, img_ph, pg_ph)\n print(json.dumps({\"ok\": True, \"mode\": \"markdown\", \"text\": text, \"meta\": meta}))\n return\n\n # structured\n try:\n doc_dict = doc.export_to_dict()\n except Exception as e:\n print(json.dumps({\"ok\": False, \"error\": f\"Docling export_to_dict failed: {e}\", \"meta\": meta}))\n return\n\n rows = to_rows(doc_dict)\n print(json.dumps({\"ok\": True, \"mode\": \"structured\", \"doc\": rows, \"meta\": meta}))\n except Exception as e:\n print(\n json.dumps({\n \"ok\": False,\n \"error\": f\"Docling processing error: {e}\",\n \"meta\": {\"file_path\": file_path},\n })\n )\n\n if __name__ == \"__main__\":\n main()\n \"\"\"\n )\n\n # Validate file_path to avoid command injection or unsafe input\n if not isinstance(args[\"file_path\"], str) or any(c in args[\"file_path\"] for c in [\";\", \"|\", \"&\", \"$\", \"`\"]):\n return Data(data={\"error\": \"Unsafe file path detected.\", \"file_path\": args[\"file_path\"]})\n\n proc = subprocess.run( # noqa: S603\n [sys.executable, \"-u\", \"-c\", child_script],\n input=json.dumps(args).encode(\"utf-8\"),\n capture_output=True,\n check=False,\n )\n\n if not proc.stdout:\n err_msg = proc.stderr.decode(\"utf-8\", errors=\"replace\") or \"no output from child process\"\n return Data(data={\"error\": f\"Docling subprocess error: {err_msg}\", \"file_path\": file_path})\n\n try:\n result = json.loads(proc.stdout.decode(\"utf-8\"))\n except Exception as e: # noqa: BLE001\n err_msg = proc.stderr.decode(\"utf-8\", errors=\"replace\")\n return Data(\n data={\"error\": f\"Invalid JSON from Docling subprocess: {e}. stderr={err_msg}\", \"file_path\": file_path},\n )\n\n if not result.get(\"ok\"):\n return Data(data={\"error\": result.get(\"error\", \"Unknown Docling error\"), **result.get(\"meta\", {})})\n\n meta = result.get(\"meta\", {})\n if result.get(\"mode\") == \"markdown\":\n exported_content = str(result.get(\"text\", \"\"))\n return Data(\n text=exported_content,\n data={\"exported_content\": exported_content, \"export_format\": self.EXPORT_FORMAT, **meta},\n )\n\n rows = list(result.get(\"doc\", []))\n return Data(data={\"doc\": rows, \"export_format\": self.EXPORT_FORMAT, **meta})\n\n def process_files(\n self,\n file_list: list[BaseFileComponent.BaseFile],\n ) -> list[BaseFileComponent.BaseFile]:\n \"\"\"Process input files.\n\n - Single file + advanced_mode => Docling in a separate process.\n - Otherwise => standard parsing in current process (optionally threaded).\n \"\"\"\n if not file_list:\n msg = \"No files to process.\"\n raise ValueError(msg)\n\n def process_file_standard(file_path: str, *, silent_errors: bool = False) -> Data | None:\n try:\n return parse_text_file_to_data(file_path, silent_errors=silent_errors)\n except FileNotFoundError as e:\n self.log(f\"File not found: {file_path}. Error: {e}\")\n if not silent_errors:\n raise\n return None\n except Exception as e:\n self.log(f\"Unexpected error processing {file_path}: {e}\")\n if not silent_errors:\n raise\n return None\n\n # Advanced path: only for a single Docling-compatible file\n if len(file_list) == 1:\n file_path = str(file_list[0].path)\n if self.advanced_mode and self._is_docling_compatible(file_path):\n advanced_data: Data | None = self._process_docling_in_subprocess(file_path)\n\n # --- UNNEST: expand each element in `doc` to its own Data row\n payload = getattr(advanced_data, \"data\", {}) or {}\n doc_rows = payload.get(\"doc\")\n if isinstance(doc_rows, list):\n rows: list[Data | None] = [\n Data(\n data={\n \"file_path\": file_path,\n **(item if isinstance(item, dict) else {\"value\": item}),\n },\n )\n for item in doc_rows\n ]\n return self.rollup_data(file_list, rows)\n\n # If not structured, keep as-is (e.g., markdown export or error dict)\n return self.rollup_data(file_list, [advanced_data])\n\n # Standard multi-file (or single non-advanced) path\n concurrency = 1 if not self.use_multithreading else max(1, self.concurrency_multithreading)\n file_paths = [str(f.path) for f in file_list]\n self.log(f\"Starting parallel processing of {len(file_paths)} files with concurrency: {concurrency}.\")\n my_data = parallel_load_data(\n file_paths,\n silent_errors=self.silent_errors,\n load_function=process_file_standard,\n max_concurrency=concurrency,\n )\n return self.rollup_data(file_list, my_data)\n\n # ------------------------------ Output helpers -----------------------------------\n\n def load_files_advanced(self) -> DataFrame:\n \"\"\"Load files using advanced Docling processing and export to an advanced format.\"\"\"\n self.markdown = False\n return self.load_files()\n\n def load_files_markdown(self) -> Message:\n \"\"\"Load files using advanced Docling processing and export to Markdown format.\"\"\"\n self.markdown = True\n result = self.load_files()\n return Message(text=str(result.text[0]))\n"
|
|
1306
1306
|
},
|
|
1307
1307
|
"concurrency_multithreading": {
|
|
1308
1308
|
"_input_type": "IntInput",
|
|
@@ -724,7 +724,7 @@
|
|
|
724
724
|
},
|
|
725
725
|
{
|
|
726
726
|
"name": "fastapi",
|
|
727
|
-
"version": "0.
|
|
727
|
+
"version": "0.116.1"
|
|
728
728
|
},
|
|
729
729
|
{
|
|
730
730
|
"name": "langflow",
|
|
@@ -1255,7 +1255,7 @@
|
|
|
1255
1255
|
},
|
|
1256
1256
|
{
|
|
1257
1257
|
"name": "langchain_core",
|
|
1258
|
-
"version": "0.3.
|
|
1258
|
+
"version": "0.3.75"
|
|
1259
1259
|
},
|
|
1260
1260
|
{
|
|
1261
1261
|
"name": "langflow",
|
|
@@ -1091,7 +1091,7 @@
|
|
|
1091
1091
|
},
|
|
1092
1092
|
{
|
|
1093
1093
|
"name": "fastapi",
|
|
1094
|
-
"version": "0.
|
|
1094
|
+
"version": "0.116.1"
|
|
1095
1095
|
},
|
|
1096
1096
|
{
|
|
1097
1097
|
"name": "langflow",
|
|
@@ -1627,7 +1627,7 @@
|
|
|
1627
1627
|
"dependencies": [
|
|
1628
1628
|
{
|
|
1629
1629
|
"name": "httpx",
|
|
1630
|
-
"version": "0.
|
|
1630
|
+
"version": "0.28.1"
|
|
1631
1631
|
},
|
|
1632
1632
|
{
|
|
1633
1633
|
"name": "langflow",
|
|
@@ -93,7 +93,7 @@
|
|
|
93
93
|
"dependencies": [
|
|
94
94
|
{
|
|
95
95
|
"name": "langchain_text_splitters",
|
|
96
|
-
"version": "0.3.
|
|
96
|
+
"version": "0.3.9"
|
|
97
97
|
},
|
|
98
98
|
{
|
|
99
99
|
"name": "langflow",
|
|
@@ -357,7 +357,7 @@
|
|
|
357
357
|
"dependencies": [
|
|
358
358
|
{
|
|
359
359
|
"name": "requests",
|
|
360
|
-
"version": "2.32.
|
|
360
|
+
"version": "2.32.5"
|
|
361
361
|
},
|
|
362
362
|
{
|
|
363
363
|
"name": "bs4",
|
|
@@ -515,7 +515,7 @@
|
|
|
515
515
|
},
|
|
516
516
|
{
|
|
517
517
|
"name": "fastapi",
|
|
518
|
-
"version": "0.
|
|
518
|
+
"version": "0.116.1"
|
|
519
519
|
},
|
|
520
520
|
{
|
|
521
521
|
"name": "langflow",
|
|
@@ -1238,7 +1238,7 @@
|
|
|
1238
1238
|
"dependencies": [
|
|
1239
1239
|
{
|
|
1240
1240
|
"name": "httpx",
|
|
1241
|
-
"version": "0.
|
|
1241
|
+
"version": "0.28.1"
|
|
1242
1242
|
},
|
|
1243
1243
|
{
|
|
1244
1244
|
"name": "langflow",
|
|
@@ -648,7 +648,7 @@
|
|
|
648
648
|
},
|
|
649
649
|
{
|
|
650
650
|
"name": "fastapi",
|
|
651
|
-
"version": "0.
|
|
651
|
+
"version": "0.116.1"
|
|
652
652
|
},
|
|
653
653
|
{
|
|
654
654
|
"name": "langflow",
|
|
@@ -970,7 +970,7 @@
|
|
|
970
970
|
},
|
|
971
971
|
{
|
|
972
972
|
"name": "fastapi",
|
|
973
|
-
"version": "0.
|
|
973
|
+
"version": "0.116.1"
|
|
974
974
|
},
|
|
975
975
|
{
|
|
976
976
|
"name": "langflow",
|
|
@@ -1292,7 +1292,7 @@
|
|
|
1292
1292
|
},
|
|
1293
1293
|
{
|
|
1294
1294
|
"name": "fastapi",
|
|
1295
|
-
"version": "0.
|
|
1295
|
+
"version": "0.116.1"
|
|
1296
1296
|
},
|
|
1297
1297
|
{
|
|
1298
1298
|
"name": "langflow",
|
|
@@ -210,7 +210,7 @@
|
|
|
210
210
|
"dependencies": [
|
|
211
211
|
{
|
|
212
212
|
"name": "httpx",
|
|
213
|
-
"version": "0.
|
|
213
|
+
"version": "0.28.1"
|
|
214
214
|
},
|
|
215
215
|
{
|
|
216
216
|
"name": "langflow",
|
|
@@ -934,7 +934,7 @@
|
|
|
934
934
|
},
|
|
935
935
|
{
|
|
936
936
|
"name": "fastapi",
|
|
937
|
-
"version": "0.
|
|
937
|
+
"version": "0.116.1"
|
|
938
938
|
},
|
|
939
939
|
{
|
|
940
940
|
"name": "langflow",
|
|
@@ -1260,7 +1260,7 @@
|
|
|
1260
1260
|
},
|
|
1261
1261
|
{
|
|
1262
1262
|
"name": "fastapi",
|
|
1263
|
-
"version": "0.
|
|
1263
|
+
"version": "0.116.1"
|
|
1264
1264
|
},
|
|
1265
1265
|
{
|
|
1266
1266
|
"name": "langflow",
|
|
@@ -566,7 +566,7 @@
|
|
|
566
566
|
},
|
|
567
567
|
{
|
|
568
568
|
"name": "fastapi",
|
|
569
|
-
"version": "0.
|
|
569
|
+
"version": "0.116.1"
|
|
570
570
|
},
|
|
571
571
|
{
|
|
572
572
|
"name": "langflow",
|
|
@@ -2575,7 +2575,7 @@
|
|
|
2575
2575
|
"dependencies": [
|
|
2576
2576
|
{
|
|
2577
2577
|
"name": "langchain_core",
|
|
2578
|
-
"version": "0.3.
|
|
2578
|
+
"version": "0.3.75"
|
|
2579
2579
|
},
|
|
2580
2580
|
{
|
|
2581
2581
|
"name": "langflow",
|
|
@@ -447,7 +447,7 @@
|
|
|
447
447
|
},
|
|
448
448
|
{
|
|
449
449
|
"name": "fastapi",
|
|
450
|
-
"version": "0.
|
|
450
|
+
"version": "0.116.1"
|
|
451
451
|
},
|
|
452
452
|
{
|
|
453
453
|
"name": "langflow",
|
|
@@ -866,7 +866,7 @@
|
|
|
866
866
|
},
|
|
867
867
|
{
|
|
868
868
|
"name": "httpx",
|
|
869
|
-
"version": "0.
|
|
869
|
+
"version": "0.28.1"
|
|
870
870
|
},
|
|
871
871
|
{
|
|
872
872
|
"name": "validators",
|