lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/processing/dynamic_create_data.py

@@ -0,0 +1,357 @@
+from typing import Any
+
+from lfx.custom import Component
+from lfx.io import (
+    BoolInput,
+    FloatInput,
+    HandleInput,
+    IntInput,
+    MultilineInput,
+    Output,
+    StrInput,
+    TableInput,
+)
+from lfx.schema.data import Data
+from lfx.schema.message import Message
+
+
+class DynamicCreateDataComponent(Component):
+    display_name: str = "Dynamic Create Data"
+    description: str = "Dynamically create a Data with a specified number of fields."
+    name: str = "DynamicCreateData"
+    MAX_FIELDS = 15  # Define a constant for maximum number of fields
+    icon = "ListFilter"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    inputs = [
+        TableInput(
+            name="form_fields",
+            display_name="Input Configuration",
+            info=(
+                "Define the dynamic form fields. Each row creates a new input field "
+                "that can connect to other components."
+            ),
+            table_schema=[
+                {
+                    "name": "field_name",
+                    "display_name": "Field Name",
+                    "type": "str",
+                    "description": "Name for the field (used as both internal name and display label)",
+                },
+                {
+                    "name": "field_type",
+                    "display_name": "Field Type",
+                    "type": "str",
+                    "description": "Type of input field to create",
+                    "options": ["Text", "Data", "Number", "Handle", "Boolean"],
+                    "value": "Text",
+                },
+            ],
+            value=[],
+            real_time_refresh=True,
+        ),
+        BoolInput(
+            name="include_metadata",
+            display_name="Include Metadata",
+            info="Include form configuration metadata in the output.",
+            value=False,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="form_data", method="process_form"),
+        Output(display_name="Message", name="message", method="get_message"),
+    ]
+
+    def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:
+        """Update build configuration to add dynamic inputs that can connect to other components."""
+        if field_name == "form_fields":
+            # Clear existing dynamic inputs from build config
+            keys_to_remove = [key for key in build_config if key.startswith("dynamic_")]
+            for key in keys_to_remove:
+                del build_config[key]
+
+            # Add dynamic inputs based on table configuration
+            # Safety check to ensure field_value is not None and is iterable
+            if field_value is None:
+                field_value = []
+
+            for i, field_config in enumerate(field_value):
+                # Safety check to ensure field_config is not None
+                if field_config is None:
+                    continue
+
+                field_name = field_config.get("field_name", f"field_{i}")
+                display_name = field_name  # Use field_name as display_name
+                field_type_option = field_config.get("field_type", "Text")
+                default_value = ""  # All fields have empty default value
+                required = False  # All fields are optional by default
+                help_text = ""  # All fields have empty help text
+
+                # Map field type options to actual field types and input types
+                field_type_mapping = {
+                    "Text": {"field_type": "multiline", "input_types": ["Text", "Message"]},
+                    "Data": {"field_type": "data", "input_types": ["Data"]},
+                    "Number": {"field_type": "number", "input_types": ["Text", "Message"]},
+                    "Handle": {"field_type": "handle", "input_types": ["Text", "Data", "Message"]},
+                    "Boolean": {"field_type": "boolean", "input_types": None},
+                }
+
+                field_config_mapped = field_type_mapping.get(
+                    field_type_option, {"field_type": "text", "input_types": []}
+                )
+                if not isinstance(field_config_mapped, dict):
+                    field_config_mapped = {"field_type": "text", "input_types": []}
+                field_type = field_config_mapped["field_type"]
+                input_types_list = field_config_mapped["input_types"]
+
+                # Create the appropriate input type based on field_type
+                dynamic_input_name = f"dynamic_{field_name}"
+
+                if field_type == "text":
+                    if input_types_list:
+                        build_config[dynamic_input_name] = StrInput(
+                            name=dynamic_input_name,
+                            display_name=display_name,
+                            info=f"{help_text} (Can connect to: {', '.join(input_types_list)})",
+                            value=default_value,
+                            required=required,
+                            input_types=input_types_list,
+                        )
+                    else:
+                        build_config[dynamic_input_name] = StrInput(
+                            name=dynamic_input_name,
+                            display_name=display_name,
+                            info=help_text,
+                            value=default_value,
+                            required=required,
+                        )
+
+                elif field_type == "multiline":
+                    if input_types_list:
+                        build_config[dynamic_input_name] = MultilineInput(
+                            name=dynamic_input_name,
+                            display_name=display_name,
+                            info=f"{help_text} (Can connect to: {', '.join(input_types_list)})",
+                            value=default_value,
+                            required=required,
+                            input_types=input_types_list,
+                        )
+                    else:
+                        build_config[dynamic_input_name] = MultilineInput(
+                            name=dynamic_input_name,
+                            display_name=display_name,
+                            info=help_text,
+                            value=default_value,
+                            required=required,
+                        )
+
+                elif field_type == "number":
+                    try:
+                        default_int = int(default_value) if default_value else 0
+                    except ValueError:
+                        default_int = 0
+
+                    if input_types_list:
+                        build_config[dynamic_input_name] = IntInput(
+                            name=dynamic_input_name,
+                            display_name=display_name,
+                            info=f"{help_text} (Can connect to: {', '.join(input_types_list)})",
+                            value=default_int,
+                            required=required,
+                            input_types=input_types_list,
+                        )
+                    else:
+                        build_config[dynamic_input_name] = IntInput(
+                            name=dynamic_input_name,
+                            display_name=display_name,
+                            info=help_text,
+                            value=default_int,
+                            required=required,
+                        )
+
+                elif field_type == "float":
+                    try:
+                        default_float = float(default_value) if default_value else 0.0
+                    except ValueError:
+                        default_float = 0.0
+
+                    if input_types_list:
+                        build_config[dynamic_input_name] = FloatInput(
+                            name=dynamic_input_name,
+                            display_name=display_name,
+                            info=f"{help_text} (Can connect to: {', '.join(input_types_list)})",
+                            value=default_float,
+                            required=required,
+                            input_types=input_types_list,
+                        )
+                    else:
+                        build_config[dynamic_input_name] = FloatInput(
+                            name=dynamic_input_name,
+                            display_name=display_name,
+                            info=help_text,
+                            value=default_float,
+                            required=required,
+                        )
+
+                elif field_type == "boolean":
+                    default_bool = default_value.lower() in ["true", "1", "yes"] if default_value else False
+
+                    # Boolean fields don't use input_types parameter to avoid errors
+                    build_config[dynamic_input_name] = BoolInput(
+                        name=dynamic_input_name,
+                        display_name=display_name,
+                        info=help_text,
+                        value=default_bool,
+                        input_types=[],
+                        required=required,
+                    )
+
+                elif field_type == "handle":
+                    # HandleInput for generic data connections
+                    build_config[dynamic_input_name] = HandleInput(
+                        name=dynamic_input_name,
+                        display_name=display_name,
+                        info=f"{help_text} (Accepts: {', '.join(input_types_list) if input_types_list else 'Any'})",
+                        input_types=input_types_list if input_types_list else ["Data", "Text", "Message"],
+                        required=required,
+                    )
+
+                elif field_type == "data":
+                    # Specialized for Data type connections
+                    build_config[dynamic_input_name] = HandleInput(
+                        name=dynamic_input_name,
+                        display_name=display_name,
+                        info=f"{help_text} (Data input)",
+                        input_types=input_types_list if input_types_list else ["Data"],
+                        required=required,
+                    )
+
+                else:
+                    # Default to text input for unknown types
+                    build_config[dynamic_input_name] = StrInput(
+                        name=dynamic_input_name,
+                        display_name=display_name,
+                        info=f"{help_text} (Unknown type '{field_type}', defaulting to text)",
+                        value=default_value,
+                        required=required,
+                    )
+
+        return build_config
+
+    def get_dynamic_values(self) -> dict[str, Any]:
+        """Extract simple values from all dynamic inputs, handling both manual and connected inputs."""
+        dynamic_values = {}
+        connection_info = {}
+        form_fields = getattr(self, "form_fields", [])
+
+        for field_config in form_fields:
+            # Safety check to ensure field_config is not None
+            if field_config is None:
+                continue
+
+            field_name = field_config.get("field_name", "")
+            if field_name:
+                dynamic_input_name = f"dynamic_{field_name}"
+                value = getattr(self, dynamic_input_name, None)
+
+                # Extract simple values from connections or manual input
+                if value is not None:
+                    try:
+                        extracted_value = self._extract_simple_value(value)
+                        dynamic_values[field_name] = extracted_value
+
+                        # Determine connection type for status
+                        if hasattr(value, "text") and hasattr(value, "timestamp"):
+                            connection_info[field_name] = "Connected (Message)"
+                        elif hasattr(value, "data"):
+                            connection_info[field_name] = "Connected (Data)"
+                        elif isinstance(value, (str, int, float, bool, list, dict)):
+                            connection_info[field_name] = "Manual input"
+                        else:
+                            connection_info[field_name] = "Connected (Object)"
+
+                    except (AttributeError, TypeError, ValueError):
+                        # Fallback to string representation if all else fails
+                        dynamic_values[field_name] = str(value)
+                        connection_info[field_name] = "Error"
+                else:
+                    # Use empty default value if nothing connected
+                    dynamic_values[field_name] = ""
+                    connection_info[field_name] = "Empty default"
+
+        # Store connection info for status output
+        self._connection_info = connection_info
+        return dynamic_values
+
+    def _extract_simple_value(self, value: Any) -> Any:
+        """Extract the simplest, most useful value from any input type."""
+        # Handle None
+        if value is None:
+            return None
+
+        # Handle simple types directly
+        if isinstance(value, (str, int, float, bool)):
+            return value
+
+        # Handle lists and tuples - keep simple
+        if isinstance(value, (list, tuple)):
+            return [self._extract_simple_value(item) for item in value]
+
+        # Handle dictionaries - keep simple
+        if isinstance(value, dict):
+            return {str(k): self._extract_simple_value(v) for k, v in value.items()}
+
+        # Handle Message objects - extract only the text
+        if hasattr(value, "text"):
+            return str(value.text) if value.text is not None else ""
+
+        # Handle Data objects - extract the data content
+        if hasattr(value, "data") and value.data is not None:
+            return self._extract_simple_value(value.data)
+
+        # For any other object, convert to string
+        return str(value)
+
+    def process_form(self) -> Data:
+        """Process all dynamic form inputs and return clean data with just field values."""
+        # Get all dynamic values (just the key:value pairs)
+        dynamic_values = self.get_dynamic_values()
+
+        # Update status with connection info
+        connected_fields = len([v for v in getattr(self, "_connection_info", {}).values() if "Connected" in v])
+        total_fields = len(dynamic_values)
+
+        self.status = f"Form processed successfully. {connected_fields}/{total_fields} fields connected to components."
+
+        # Return clean Data object with just the field values
+        return Data(data=dynamic_values)
+
+    def get_message(self) -> Message:
+        """Return form data as a formatted text message."""
+        # Get all dynamic values
+        dynamic_values = self.get_dynamic_values()
+
+        if not dynamic_values:
+            return Message(text="No form data available")
+
+        # Format as text message
+        message_lines = ["📋 Form Data:"]
+        message_lines.append("=" * 40)
+
+        for field_name, value in dynamic_values.items():
+            # Use field_name as display_name
+            display_name = field_name
+
+            message_lines.append(f"• {display_name}: {value}")
+
+        message_lines.append("=" * 40)
+        message_lines.append(f"Total fields: {len(dynamic_values)}")
+
+        message_text = "\n".join(message_lines)
+        self.status = f"Message formatted with {len(dynamic_values)} fields"
+
+        return Message(text=message_text)

lfx/components/processing/extract_key.py

@@ -0,0 +1,54 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, Output, StrInput
+from lfx.schema.data import Data
+
+
+class ExtractDataKeyComponent(Component):
+    display_name = "Extract Key"
+    description = (
+        "Extract a specific key from a Data object or a list of "
+        "Data objects and return the extracted value(s) as Data object(s)."
+    )
+    icon = "key"
+    name = "ExtractaKey"
+    legacy = True
+    replacement = ["processing.DataOperations"]
+
+    inputs = [
+        DataInput(
+            name="data_input",
+            display_name="Data Input",
+            info="The Data object or list of Data objects to extract the key from.",
+        ),
+        StrInput(
+            name="key",
+            display_name="Key to Extract",
+            info="The key in the Data object(s) to extract.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Extracted Data", name="extracted_data", method="extract_key"),
+    ]
+
+    def extract_key(self) -> Data | list[Data]:
+        key = self.key
+
+        if isinstance(self.data_input, list):
+            result = []
+            for item in self.data_input:
+                if isinstance(item, Data) and key in item.data:
+                    extracted_value = item.data[key]
+                    result.append(Data(data={key: extracted_value}))
+            self.status = result
+            return result
+        if isinstance(self.data_input, Data):
+            if key in self.data_input.data:
+                extracted_value = self.data_input.data[key]
+                result = Data(data={key: extracted_value})
+                self.status = result
+                return result
+            self.status = f"Key '{key}' not found in Data object."
+            return Data(data={"error": f"Key '{key}' not found in Data object."})
+        self.status = "Invalid input. Expected Data object or list of Data objects."
+        return Data(data={"error": "Invalid input. Expected Data object or list of Data objects."})

lfx/components/processing/filter_data.py

@@ -0,0 +1,43 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, MessageTextInput, Output
+from lfx.schema.data import Data
+
+
+class FilterDataComponent(Component):
+    display_name = "Filter Data"
+    description = "Filters a Data object based on a list of keys."
+    icon = "filter"
+    beta = True
+    name = "FilterData"
+    legacy = True
+    replacement = ["processing.DataOperations"]
+
+    inputs = [
+        DataInput(
+            name="data",
+            display_name="Data",
+            info="Data object to filter.",
+        ),
+        MessageTextInput(
+            name="filter_criteria",
+            display_name="Filter Criteria",
+            info="List of keys to filter by.",
+            is_list=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Filtered Data", name="filtered_data", method="filter_data"),
+    ]
+
+    def filter_data(self) -> Data:
+        filter_criteria: list[str] = self.filter_criteria
+        data = self.data.data if isinstance(self.data, Data) else {}
+
+        # Filter the data
+        filtered = {key: value for key, value in data.items() if key in filter_criteria}
+
+        # Create a new Data object with the filtered data
+        filtered_data = Data(data=filtered)
+        self.status = filtered_data
+        return filtered_data

lfx/components/processing/filter_data_values.py

@@ -0,0 +1,89 @@
+from typing import Any
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, DropdownInput, MessageTextInput, Output
+from lfx.schema.data import Data
+
+
+class DataFilterComponent(Component):
+    display_name = "Filter Values"
+    description = (
+        "Filter a list of data items based on a specified key, filter value,"
+        " and comparison operator. Check advanced options to select match comparision."
+    )
+    icon = "filter"
+    beta = True
+    name = "FilterDataValues"
+    legacy = True
+    replacement = ["processing.DataOperations"]
+
+    inputs = [
+        DataInput(name="input_data", display_name="Input Data", info="The list of data items to filter.", is_list=True),
+        MessageTextInput(
+            name="filter_key",
+            display_name="Filter Key",
+            info="The key to filter on (e.g., 'route').",
+            value="route",
+            input_types=["Data"],
+        ),
+        MessageTextInput(
+            name="filter_value",
+            display_name="Filter Value",
+            info="The value to filter by (e.g., 'CMIP').",
+            value="CMIP",
+            input_types=["Data"],
+        ),
+        DropdownInput(
+            name="operator",
+            display_name="Comparison Operator",
+            options=["equals", "not equals", "contains", "starts with", "ends with"],
+            info="The operator to apply for comparing the values.",
+            value="equals",
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Filtered Data", name="filtered_data", method="filter_data"),
+    ]
+
+    def compare_values(self, item_value: Any, filter_value: str, operator: str) -> bool:
+        if operator == "equals":
+            return str(item_value) == filter_value
+        if operator == "not equals":
+            return str(item_value) != filter_value
+        if operator == "contains":
+            return filter_value in str(item_value)
+        if operator == "starts with":
+            return str(item_value).startswith(filter_value)
+        if operator == "ends with":
+            return str(item_value).endswith(filter_value)
+        return False
+
+    def filter_data(self) -> list[Data]:
+        # Extract inputs
+        input_data: list[Data] = self.input_data
+        filter_key: str = self.filter_key.text
+        filter_value: str = self.filter_value.text
+        operator: str = self.operator
+
+        # Validate inputs
+        if not input_data:
+            self.status = "Input data is empty."
+            return []
+
+        if not filter_key or not filter_value:
+            self.status = "Filter key or value is missing."
+            return input_data
+
+        # Filter the data
+        filtered_data = []
+        for item in input_data:
+            if isinstance(item.data, dict) and filter_key in item.data:
+                if self.compare_values(item.data[filter_key], filter_value, operator):
+                    filtered_data.append(item)
+            else:
+                self.status = f"Warning: Some items don't have the key '{filter_key}' or are not dictionaries."
+
+        self.status = filtered_data
+        return filtered_data

lfx/components/processing/json_cleaner.py

@@ -0,0 +1,104 @@
+import json
+import unicodedata
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import BoolInput, MessageTextInput
+from lfx.schema.message import Message
+from lfx.template.field.base import Output
+
+
+class JSONCleaner(Component):
+    icon = "braces"
+    display_name = "JSON Cleaner"
+    description = (
+        "Cleans the messy and sometimes incorrect JSON strings produced by LLMs "
+        "so that they are fully compliant with the JSON spec."
+    )
+    legacy = True
+    replacement = ["processing.ParserComponent"]
+    inputs = [
+        MessageTextInput(
+            name="json_str", display_name="JSON String", info="The JSON string to be cleaned.", required=True
+        ),
+        BoolInput(
+            name="remove_control_chars",
+            display_name="Remove Control Characters",
+            info="Remove control characters from the JSON string.",
+            required=False,
+        ),
+        BoolInput(
+            name="normalize_unicode",
+            display_name="Normalize Unicode",
+            info="Normalize Unicode characters in the JSON string.",
+            required=False,
+        ),
+        BoolInput(
+            name="validate_json",
+            display_name="Validate JSON",
+            info="Validate the JSON string to ensure it is well-formed.",
+            required=False,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Cleaned JSON String", name="output", method="clean_json"),
+    ]
+
+    def clean_json(self) -> Message:
+        try:
+            from json_repair import repair_json
+        except ImportError as e:
+            msg = "Could not import the json_repair package. Please install it with `pip install json_repair`."
+            raise ImportError(msg) from e
+
+        """Clean the input JSON string based on provided options and return the cleaned JSON string."""
+        json_str = self.json_str
+        remove_control_chars = self.remove_control_chars
+        normalize_unicode = self.normalize_unicode
+        validate_json = self.validate_json
+
+        start = json_str.find("{")
+        end = json_str.rfind("}")
+        if start == -1 or end == -1:
+            msg = "Invalid JSON string: Missing '{' or '}'"
+            raise ValueError(msg)
+        try:
+            json_str = json_str[start : end + 1]
+
+            if remove_control_chars:
+                json_str = self._remove_control_characters(json_str)
+            if normalize_unicode:
+                json_str = self._normalize_unicode(json_str)
+            if validate_json:
+                json_str = self._validate_json(json_str)
+
+            cleaned_json_str = repair_json(json_str)
+            result = str(cleaned_json_str)
+
+            self.status = result
+            return Message(text=result)
+        except Exception as e:
+            msg = f"Error cleaning JSON string: {e}"
+            raise ValueError(msg) from e
+
+    def _remove_control_characters(self, s: str) -> str:
+        """Remove control characters from the string."""
+        return s.translate(self.translation_table)
+
+    def _normalize_unicode(self, s: str) -> str:
+        """Normalize Unicode characters in the string."""
+        return unicodedata.normalize("NFC", s)
+
+    def _validate_json(self, s: str) -> str:
+        """Validate the JSON string."""
+        try:
+            json.loads(s)
+        except json.JSONDecodeError as e:
+            msg = f"Invalid JSON string: {e}"
+            raise ValueError(msg) from e
+        return s
+
+    def __init__(self, *args, **kwargs):
+        # Create a translation table that maps control characters to None
+        super().__init__(*args, **kwargs)
+        self.translation_table = str.maketrans("", "", "".join(chr(i) for i in range(32)) + chr(127))