lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/processing/merge_data.py

@@ -0,0 +1,91 @@
+from enum import Enum
+from typing import cast
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, DropdownInput, Output
+from lfx.log.logger import logger
+from lfx.schema.dataframe import DataFrame
+
+
+class DataOperation(str, Enum):
+    CONCATENATE = "Concatenate"
+    APPEND = "Append"
+    MERGE = "Merge"
+    JOIN = "Join"
+
+
+class MergeDataComponent(Component):
+    display_name = "Combine Data"
+    description = "Combines data using different operations"
+    icon = "merge"
+    MIN_INPUTS_REQUIRED = 2
+    legacy = True
+    replacement = ["processing.DataOperations"]
+
+    inputs = [
+        DataInput(name="data_inputs", display_name="Data Inputs", info="Data to combine", is_list=True, required=True),
+        DropdownInput(
+            name="operation",
+            display_name="Operation Type",
+            options=[op.value for op in DataOperation],
+            value=DataOperation.CONCATENATE.value,
+        ),
+    ]
+    outputs = [Output(display_name="DataFrame", name="combined_data", method="combine_data")]
+
+    def combine_data(self) -> DataFrame:
+        if not self.data_inputs or len(self.data_inputs) < self.MIN_INPUTS_REQUIRED:
+            empty_dataframe = DataFrame()
+            self.status = empty_dataframe
+            return empty_dataframe
+
+        operation = DataOperation(self.operation)
+        try:
+            combined_dataframe = self._process_operation(operation)
+            self.status = combined_dataframe
+        except Exception as e:
+            logger.error(f"Error during operation {operation}: {e!s}")
+            raise
+        else:
+            return combined_dataframe
+
+    def _process_operation(self, operation: DataOperation) -> DataFrame:
+        if operation == DataOperation.CONCATENATE:
+            combined_data: dict[str, str | object] = {}
+            for data_input in self.data_inputs:
+                for key, value in data_input.data.items():
+                    if key in combined_data:
+                        if isinstance(combined_data[key], str) and isinstance(value, str):
+                            combined_data[key] = f"{combined_data[key]}\n{value}"
+                        else:
+                            combined_data[key] = value
+                    else:
+                        combined_data[key] = value
+            return DataFrame([combined_data])
+
+        if operation == DataOperation.APPEND:
+            rows = [data_input.data for data_input in self.data_inputs]
+            return DataFrame(rows)
+
+        if operation == DataOperation.MERGE:
+            result_data: dict[str, str | list[str] | object] = {}
+            for data_input in self.data_inputs:
+                for key, value in data_input.data.items():
+                    if key in result_data and isinstance(value, str):
+                        if isinstance(result_data[key], list):
+                            cast("list[str]", result_data[key]).append(value)
+                        else:
+                            result_data[key] = [result_data[key], value]
+                    else:
+                        result_data[key] = value
+            return DataFrame([result_data])
+
+        if operation == DataOperation.JOIN:
+            combined_data = {}
+            for idx, data_input in enumerate(self.data_inputs, 1):
+                for key, value in data_input.data.items():
+                    new_key = f"{key}_doc{idx}" if idx > 1 else key
+                    combined_data[new_key] = value
+            return DataFrame([combined_data])
+
+        return DataFrame()

lfx/components/processing/message_to_data.py

@@ -0,0 +1,37 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import MessageInput, Output
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+
+
+class MessageToDataComponent(Component):
+    display_name = "Message to Data"
+    description = "Convert a Message object to a Data object"
+    icon = "message-square-share"
+    beta = True
+    name = "MessagetoData"
+    legacy = True
+    replacement = ["processing.TypeConverterComponent"]
+
+    inputs = [
+        MessageInput(
+            name="message",
+            display_name="Message",
+            info="The Message object to convert to a Data object",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="convert_message_to_data"),
+    ]
+
+    def convert_message_to_data(self) -> Data:
+        # Check for Message by checking if it has the expected attributes instead of isinstance
+        if hasattr(self.message, "data") and hasattr(self.message, "text") and hasattr(self.message, "get_text"):
+            # Convert Message to Data - this works for both langflow.Message and lfx.Message
+            return Data(data=self.message.data)
+
+        msg = "Error converting Message to Data: Input must be a Message object"
+        logger.debug(msg, exc_info=True)
+        self.status = msg
+        return Data(data={"error": msg})

lfx/components/processing/output_parser.py

@@ -0,0 +1,46 @@
+from langchain_core.output_parsers import CommaSeparatedListOutputParser
+
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing.constants import OutputParser
+from lfx.io import DropdownInput, Output
+from lfx.schema.message import Message
+
+
+class OutputParserComponent(Component):
+    display_name = "Output Parser"
+    description = "Transforms the output of an LLM into a specified format."
+    icon = "type"
+    name = "OutputParser"
+    legacy = True
+    replacement = ["processing.StructuredOutput", "processing.ParserComponent"]
+
+    inputs = [
+        DropdownInput(
+            name="parser_type",
+            display_name="Parser",
+            options=["CSV"],
+            value="CSV",
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Format Instructions",
+            name="format_instructions",
+            info="Pass to a prompt template to include formatting instructions for LLM responses.",
+            method="format_instructions",
+        ),
+        Output(display_name="Output Parser", name="output_parser", method="build_parser"),
+    ]
+
+    def build_parser(self) -> OutputParser:
+        if self.parser_type == "CSV":
+            return CommaSeparatedListOutputParser()
+        msg = "Unsupported or missing parser"
+        raise ValueError(msg)
+
+    def format_instructions(self) -> Message:
+        if self.parser_type == "CSV":
+            return Message(text=CommaSeparatedListOutputParser().get_format_instructions())
+        msg = "Unsupported or missing parser"
+        raise ValueError(msg)

lfx/components/processing/parse_data.py

@@ -0,0 +1,71 @@
+from lfx.custom.custom_component.component import Component
+from lfx.helpers.data import data_to_text, data_to_text_list
+from lfx.io import DataInput, MultilineInput, Output, StrInput
+from lfx.schema.data import Data
+from lfx.schema.message import Message
+
+
+class ParseDataComponent(Component):
+    display_name = "Data to Message"
+    description = "Convert Data objects into Messages using any {field_name} from input data."
+    icon = "message-square"
+    name = "ParseData"
+    legacy = True
+    replacement = ["processing.DataOperations", "processing.TypeConverterComponent"]
+    metadata = {
+        "legacy_name": "Parse Data",
+    }
+
+    inputs = [
+        DataInput(
+            name="data",
+            display_name="Data",
+            info="The data to convert to text.",
+            is_list=True,
+            required=True,
+        ),
+        MultilineInput(
+            name="template",
+            display_name="Template",
+            info="The template to use for formatting the data. "
+            "It can contain the keys {text}, {data} or any other key in the Data.",
+            value="{text}",
+            required=True,
+        ),
+        StrInput(name="sep", display_name="Separator", advanced=True, value="\n"),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Message",
+            name="text",
+            info="Data as a single Message, with each input Data separated by Separator",
+            method="parse_data",
+        ),
+        Output(
+            display_name="Data List",
+            name="data_list",
+            info="Data as a list of new Data, each having `text` formatted by Template",
+            method="parse_data_as_list",
+        ),
+    ]
+
+    def _clean_args(self) -> tuple[list[Data], str, str]:
+        data = self.data if isinstance(self.data, list) else [self.data]
+        template = self.template
+        sep = self.sep
+        return data, template, sep
+
+    def parse_data(self) -> Message:
+        data, template, sep = self._clean_args()
+        result_string = data_to_text(template, data, sep)
+        self.status = result_string
+        return Message(text=result_string)
+
+    def parse_data_as_list(self) -> list[Data]:
+        data, template, _ = self._clean_args()
+        text_list, data_list = data_to_text_list(template, data)
+        for item, text in zip(data_list, text_list, strict=True):
+            item.set_text(text)
+        self.status = data_list
+        return data_list

lfx/components/processing/parse_dataframe.py

@@ -0,0 +1,69 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataFrameInput, MultilineInput, Output, StrInput
+from lfx.schema.message import Message
+
+
+class ParseDataFrameComponent(Component):
+    display_name = "Parse DataFrame"
+    description = (
+        "Convert a DataFrame into plain text following a specified template. "
+        "Each column in the DataFrame is treated as a possible template key, e.g. {col_name}."
+    )
+    icon = "braces"
+    name = "ParseDataFrame"
+    legacy = True
+    replacement = ["processing.DataFrameOperations", "processing.TypeConverterComponent"]
+
+    inputs = [
+        DataFrameInput(name="df", display_name="DataFrame", info="The DataFrame to convert to text rows."),
+        MultilineInput(
+            name="template",
+            display_name="Template",
+            info=(
+                "The template for formatting each row. "
+                "Use placeholders matching column names in the DataFrame, for example '{col1}', '{col2}'."
+            ),
+            value="{text}",
+        ),
+        StrInput(
+            name="sep",
+            display_name="Separator",
+            advanced=True,
+            value="\n",
+            info="String that joins all row texts when building the single Text output.",
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Text",
+            name="text",
+            info="All rows combined into a single text, each row formatted by the template and separated by `sep`.",
+            method="parse_data",
+        ),
+    ]
+
+    def _clean_args(self):
+        dataframe = self.df
+        template = self.template or "{text}"
+        sep = self.sep or "\n"
+        return dataframe, template, sep
+
+    def parse_data(self) -> Message:
+        """Converts each row of the DataFrame into a formatted string using the template.
+
+        then joins them with `sep`. Returns a single combined string as a Message.
+        """
+        dataframe, template, sep = self._clean_args()
+
+        lines = []
+        # For each row in the DataFrame, build a dict and format
+        for _, row in dataframe.iterrows():
+            row_dict = row.to_dict()
+            text_line = template.format(**row_dict)  # e.g. template="{text}", row_dict={"text": "Hello"}
+            lines.append(text_line)
+
+        # Join all lines with the provided separator
+        result_string = sep.join(lines)
+        self.status = result_string  # store in self.status for UI logs
+        return Message(text=result_string)

lfx/components/processing/parse_json_data.py

@@ -0,0 +1,91 @@
+import json
+from json import JSONDecodeError
+
+import jq
+from json_repair import repair_json
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import HandleInput, MessageTextInput
+from lfx.io import Output
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.schema.message import Message
+
+
+class ParseJSONDataComponent(Component):
+    display_name = "Parse JSON"
+    description = "Convert and extract JSON fields."
+    icon = "braces"
+    name = "ParseJSONData"
+    legacy: bool = True
+    replacement = ["processing.ParserComponent"]
+
+    inputs = [
+        HandleInput(
+            name="input_value",
+            display_name="Input",
+            info="Data object to filter.",
+            required=True,
+            input_types=["Message", "Data"],
+        ),
+        MessageTextInput(
+            name="query",
+            display_name="JQ Query",
+            info="JQ Query to filter the data. The input is always a JSON list.",
+            required=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Filtered Data", name="filtered_data", method="filter_data"),
+    ]
+
+    def _parse_data(self, input_value) -> str:
+        if isinstance(input_value, Message) and isinstance(input_value.text, str):
+            return input_value.text
+        if isinstance(input_value, Data):
+            return json.dumps(input_value.data)
+        return str(input_value)
+
+    def filter_data(self) -> list[Data]:
+        to_filter = self.input_value
+        if not to_filter:
+            return []
+        # Check if input is a list
+        if isinstance(to_filter, list):
+            to_filter = [self._parse_data(f) for f in to_filter]
+        else:
+            to_filter = self._parse_data(to_filter)
+
+        # If input is not a list, don't wrap it in a list
+        if not isinstance(to_filter, list):
+            to_filter = repair_json(to_filter)
+            try:
+                to_filter_as_dict = json.loads(to_filter)
+            except JSONDecodeError:
+                try:
+                    to_filter_as_dict = json.loads(repair_json(to_filter))
+                except JSONDecodeError as e:
+                    msg = f"Invalid JSON: {e}"
+                    raise ValueError(msg) from e
+        else:
+            to_filter = [repair_json(f) for f in to_filter]
+            to_filter_as_dict = []
+            for f in to_filter:
+                try:
+                    to_filter_as_dict.append(json.loads(f))
+                except JSONDecodeError:
+                    try:
+                        to_filter_as_dict.append(json.loads(repair_json(f)))
+                    except JSONDecodeError as e:
+                        msg = f"Invalid JSON: {e}"
+                        raise ValueError(msg) from e
+            to_filter = to_filter_as_dict
+
+        full_filter_str = json.dumps(to_filter_as_dict)
+
+        logger.info("to_filter: %s", to_filter)
+
+        results = jq.compile(self.query).input_text(full_filter_str).all()
+        logger.info("results: %s", results)
+        return [Data(data=value) if isinstance(value, dict) else Data(text=str(value)) for value in results]

lfx/components/processing/parser.py

@@ -0,0 +1,148 @@
+from lfx.custom.custom_component.component import Component
+from lfx.helpers.data import safe_convert
+from lfx.inputs.inputs import BoolInput, HandleInput, MessageTextInput, MultilineInput, TabInput
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.schema.message import Message
+from lfx.template.field.base import Output
+
+
+class ParserComponent(Component):
+    display_name = "Parser"
+    description = "Extracts text using a template."
+    documentation: str = "https://docs.langflow.org/parser"
+    icon = "braces"
+
+    inputs = [
+        HandleInput(
+            name="input_data",
+            display_name="Data or DataFrame",
+            input_types=["DataFrame", "Data"],
+            info="Accepts either a DataFrame or a Data object.",
+            required=True,
+        ),
+        TabInput(
+            name="mode",
+            display_name="Mode",
+            options=["Parser", "Stringify"],
+            value="Parser",
+            info="Convert into raw string instead of using a template.",
+            real_time_refresh=True,
+        ),
+        MultilineInput(
+            name="pattern",
+            display_name="Template",
+            info=(
+                "Use variables within curly brackets to extract column values for DataFrames "
+                "or key values for Data."
+                "For example: `Name: {Name}, Age: {Age}, Country: {Country}`"
+            ),
+            value="Text: {text}",  # Example default
+            dynamic=True,
+            show=True,
+            required=True,
+        ),
+        MessageTextInput(
+            name="sep",
+            display_name="Separator",
+            advanced=True,
+            value="\n",
+            info="String used to separate rows/items.",
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Parsed Text",
+            name="parsed_text",
+            info="Formatted text output.",
+            method="parse_combined_text",
+        ),
+    ]
+
+    def update_build_config(self, build_config, field_value, field_name=None):
+        """Dynamically hide/show `template` and enforce requirement based on `stringify`."""
+        if field_name == "mode":
+            build_config["pattern"]["show"] = self.mode == "Parser"
+            build_config["pattern"]["required"] = self.mode == "Parser"
+            if field_value:
+                clean_data = BoolInput(
+                    name="clean_data",
+                    display_name="Clean Data",
+                    info=(
+                        "Enable to clean the data by removing empty rows and lines "
+                        "in each cell of the DataFrame/ Data object."
+                    ),
+                    value=True,
+                    advanced=True,
+                    required=False,
+                )
+                build_config["clean_data"] = clean_data.to_dict()
+            else:
+                build_config.pop("clean_data", None)
+
+        return build_config
+
+    def _clean_args(self):
+        """Prepare arguments based on input type."""
+        input_data = self.input_data
+
+        match input_data:
+            case list() if all(isinstance(item, Data) for item in input_data):
+                msg = "List of Data objects is not supported."
+                raise ValueError(msg)
+            case DataFrame():
+                return input_data, None
+            case Data():
+                return None, input_data
+            case dict() if "data" in input_data:
+                try:
+                    if "columns" in input_data:  # Likely a DataFrame
+                        return DataFrame.from_dict(input_data), None
+                    # Likely a Data object
+                    return None, Data(**input_data)
+                except (TypeError, ValueError, KeyError) as e:
+                    msg = f"Invalid structured input provided: {e!s}"
+                    raise ValueError(msg) from e
+            case _:
+                msg = f"Unsupported input type: {type(input_data)}. Expected DataFrame or Data."
+                raise ValueError(msg)
+
+    def parse_combined_text(self) -> Message:
+        """Parse all rows/items into a single text or convert input to string if `stringify` is enabled."""
+        # Early return for stringify option
+        if self.mode == "Stringify":
+            return self.convert_to_string()
+
+        df, data = self._clean_args()
+
+        lines = []
+        if df is not None:
+            for _, row in df.iterrows():
+                formatted_text = self.pattern.format(**row.to_dict())
+                lines.append(formatted_text)
+        elif data is not None:
+            # Use format_map with a dict that returns default_value for missing keys
+            class DefaultDict(dict):
+                def __missing__(self, key):
+                    return data.default_value or ""
+
+            formatted_text = self.pattern.format_map(DefaultDict(data.data))
+            lines.append(formatted_text)
+
+        combined_text = self.sep.join(lines)
+        self.status = combined_text
+        return Message(text=combined_text)
+
+    def convert_to_string(self) -> Message:
+        """Convert input data to string with proper error handling."""
+        result = ""
+        if isinstance(self.input_data, list):
+            result = "\n".join([safe_convert(item, clean_data=self.clean_data or False) for item in self.input_data])
+        else:
+            result = safe_convert(self.input_data or False)
+        self.log(f"Converted to string with length: {len(result)}")
+
+        message = Message(text=result)
+        self.status = message
+        return message

lfx/components/processing/regex.py

@@ -0,0 +1,83 @@
+import re
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import MessageTextInput, Output
+from lfx.schema.data import Data
+from lfx.schema.message import Message
+
+
+class RegexExtractorComponent(Component):
+    display_name = "Regex Extractor"
+    description = "Extract patterns from text using regular expressions."
+    icon = "regex"
+    legacy = True
+    replacement = ["processing.ParserComponent"]
+
+    inputs = [
+        MessageTextInput(
+            name="input_text",
+            display_name="Input Text",
+            info="The text to analyze",
+            required=True,
+        ),
+        MessageTextInput(
+            name="pattern",
+            display_name="Regex Pattern",
+            info="The regular expression pattern to match",
+            value=r"",
+            required=True,
+            tool_mode=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="extract_matches"),
+        Output(display_name="Message", name="text", method="get_matches_text"),
+    ]
+
+    def extract_matches(self) -> list[Data]:
+        if not self.pattern or not self.input_text:
+            self.status = []
+            return []
+
+        try:
+            # Compile regex pattern
+            pattern = re.compile(self.pattern)
+
+            # Find all matches in the input text
+            matches = pattern.findall(self.input_text)
+
+            # Filter out empty matches
+            filtered_matches = [match for match in matches if match]  # Remove empty matches
+
+            # Return empty list for no matches, or list of matches if found
+            result: list = [] if not filtered_matches else [Data(data={"match": match}) for match in filtered_matches]
+
+        except re.error as e:
+            error_message = f"Invalid regex pattern: {e!s}"
+            result = [Data(data={"error": error_message})]
+        except ValueError as e:
+            error_message = f"Error extracting matches: {e!s}"
+            result = [Data(data={"error": error_message})]
+
+        self.status = result
+        return result
+
+    def get_matches_text(self) -> Message:
+        """Get matches as a formatted text message."""
+        matches = self.extract_matches()
+
+        if not matches:
+            message = Message(text="No matches found")
+            self.status = message
+            return message
+
+        if "error" in matches[0].data:
+            message = Message(text=matches[0].data["error"])
+            self.status = message
+            return message
+
+        result = "\n".join(match.data["match"] for match in matches)
+        message = Message(text=result)
+        self.status = message
+        return message

lfx/components/processing/select_data.py

@@ -0,0 +1,49 @@
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing.range_spec import RangeSpec
+from lfx.inputs.inputs import DataInput, IntInput
+from lfx.io import Output
+from lfx.schema.data import Data
+
+
+class SelectDataComponent(Component):
+    display_name: str = "Select Data"
+    description: str = "Select a single data from a list of data."
+    name: str = "SelectData"
+    icon = "prototypes"
+    legacy = True
+    replacement = ["processing.DataOperations"]
+
+    inputs = [
+        DataInput(
+            name="data_list",
+            display_name="Data List",
+            info="List of data to select from.",
+            is_list=True,  # Specify that this input takes a list of Data objects
+        ),
+        IntInput(
+            name="data_index",
+            display_name="Data Index",
+            info="Index of the data to select.",
+            value=0,  # Will be populated dynamically based on the length of data_list
+            range_spec=RangeSpec(min=0, max=15, step=1, step_type="int"),
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Selected Data", name="selected_data", method="select_data"),
+    ]
+
+    async def select_data(self) -> Data:
+        # Retrieve the selected index from the dropdown
+        selected_index = int(self.data_index)
+        # Get the data list
+
+        # Validate that the selected index is within bounds
+        if selected_index < 0 or selected_index >= len(self.data_list):
+            msg = f"Selected index {selected_index} is out of range."
+            raise ValueError(msg)
+
+        # Return the selected Data object
+        selected_data = self.data_list[selected_index]
+        self.status = selected_data  # Update the component status to reflect the selected data
+        return selected_data