lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/processing/dataframe_operations.py

@@ -0,0 +1,313 @@
+import pandas as pd
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs import SortableListInput
+from lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput
+from lfx.log.logger import logger
+from lfx.schema.dataframe import DataFrame
+
+
+class DataFrameOperationsComponent(Component):
+    display_name = "DataFrame Operations"
+    description = "Perform various operations on a DataFrame."
+    documentation: str = "https://docs.langflow.org/components-processing#dataframe-operations"
+    icon = "table"
+    name = "DataFrameOperations"
+
+    OPERATION_CHOICES = [
+        "Add Column",
+        "Drop Column",
+        "Filter",
+        "Head",
+        "Rename Column",
+        "Replace Value",
+        "Select Columns",
+        "Sort",
+        "Tail",
+        "Drop Duplicates",
+    ]
+
+    inputs = [
+        DataFrameInput(
+            name="df",
+            display_name="DataFrame",
+            info="The input DataFrame to operate on.",
+            required=True,
+        ),
+        SortableListInput(
+            name="operation",
+            display_name="Operation",
+            placeholder="Select Operation",
+            info="Select the DataFrame operation to perform.",
+            options=[
+                {"name": "Add Column", "icon": "plus"},
+                {"name": "Drop Column", "icon": "minus"},
+                {"name": "Filter", "icon": "filter"},
+                {"name": "Head", "icon": "arrow-up"},
+                {"name": "Rename Column", "icon": "pencil"},
+                {"name": "Replace Value", "icon": "replace"},
+                {"name": "Select Columns", "icon": "columns"},
+                {"name": "Sort", "icon": "arrow-up-down"},
+                {"name": "Tail", "icon": "arrow-down"},
+                {"name": "Drop Duplicates", "icon": "copy-x"},
+            ],
+            real_time_refresh=True,
+            limit=1,
+        ),
+        StrInput(
+            name="column_name",
+            display_name="Column Name",
+            info="The column name to use for the operation.",
+            dynamic=True,
+            show=False,
+        ),
+        MessageTextInput(
+            name="filter_value",
+            display_name="Filter Value",
+            info="The value to filter rows by.",
+            dynamic=True,
+            show=False,
+        ),
+        DropdownInput(
+            name="filter_operator",
+            display_name="Filter Operator",
+            options=[
+                "equals",
+                "not equals",
+                "contains",
+                "not contains",
+                "starts with",
+                "ends with",
+                "greater than",
+                "less than",
+            ],
+            value="equals",
+            info="The operator to apply for filtering rows.",
+            advanced=False,
+            dynamic=True,
+            show=False,
+        ),
+        BoolInput(
+            name="ascending",
+            display_name="Sort Ascending",
+            info="Whether to sort in ascending order.",
+            dynamic=True,
+            show=False,
+            value=True,
+        ),
+        StrInput(
+            name="new_column_name",
+            display_name="New Column Name",
+            info="The new column name when renaming or adding a column.",
+            dynamic=True,
+            show=False,
+        ),
+        MessageTextInput(
+            name="new_column_value",
+            display_name="New Column Value",
+            info="The value to populate the new column with.",
+            dynamic=True,
+            show=False,
+        ),
+        StrInput(
+            name="columns_to_select",
+            display_name="Columns to Select",
+            dynamic=True,
+            is_list=True,
+            show=False,
+        ),
+        IntInput(
+            name="num_rows",
+            display_name="Number of Rows",
+            info="Number of rows to return (for head/tail).",
+            dynamic=True,
+            show=False,
+            value=5,
+        ),
+        MessageTextInput(
+            name="replace_value",
+            display_name="Value to Replace",
+            info="The value to replace in the column.",
+            dynamic=True,
+            show=False,
+        ),
+        MessageTextInput(
+            name="replacement_value",
+            display_name="Replacement Value",
+            info="The value to replace with.",
+            dynamic=True,
+            show=False,
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="DataFrame",
+            name="output",
+            method="perform_operation",
+            info="The resulting DataFrame after the operation.",
+        )
+    ]
+
+    def update_build_config(self, build_config, field_value, field_name=None):
+        dynamic_fields = [
+            "column_name",
+            "filter_value",
+            "filter_operator",
+            "ascending",
+            "new_column_name",
+            "new_column_value",
+            "columns_to_select",
+            "num_rows",
+            "replace_value",
+            "replacement_value",
+        ]
+        for field in dynamic_fields:
+            build_config[field]["show"] = False
+
+        if field_name == "operation":
+            # Handle SortableListInput format
+            if isinstance(field_value, list):
+                operation_name = field_value[0].get("name", "") if field_value else ""
+            else:
+                operation_name = field_value or ""
+
+            # If no operation selected, all dynamic fields stay hidden (already set to False above)
+            if not operation_name:
+                return build_config
+
+            if operation_name == "Filter":
+                build_config["column_name"]["show"] = True
+                build_config["filter_value"]["show"] = True
+                build_config["filter_operator"]["show"] = True
+            elif operation_name == "Sort":
+                build_config["column_name"]["show"] = True
+                build_config["ascending"]["show"] = True
+            elif operation_name == "Drop Column":
+                build_config["column_name"]["show"] = True
+            elif operation_name == "Rename Column":
+                build_config["column_name"]["show"] = True
+                build_config["new_column_name"]["show"] = True
+            elif operation_name == "Add Column":
+                build_config["new_column_name"]["show"] = True
+                build_config["new_column_value"]["show"] = True
+            elif operation_name == "Select Columns":
+                build_config["columns_to_select"]["show"] = True
+            elif operation_name in {"Head", "Tail"}:
+                build_config["num_rows"]["show"] = True
+            elif operation_name == "Replace Value":
+                build_config["column_name"]["show"] = True
+                build_config["replace_value"]["show"] = True
+                build_config["replacement_value"]["show"] = True
+            elif operation_name == "Drop Duplicates":
+                build_config["column_name"]["show"] = True
+
+        return build_config
+
+    def perform_operation(self) -> DataFrame:
+        df_copy = self.df.copy()
+
+        # Handle SortableListInput format for operation
+        operation_input = getattr(self, "operation", [])
+        if isinstance(operation_input, list) and len(operation_input) > 0:
+            op = operation_input[0].get("name", "")
+        else:
+            op = ""
+
+        # If no operation selected, return original DataFrame
+        if not op:
+            return df_copy
+
+        if op == "Filter":
+            return self.filter_rows_by_value(df_copy)
+        if op == "Sort":
+            return self.sort_by_column(df_copy)
+        if op == "Drop Column":
+            return self.drop_column(df_copy)
+        if op == "Rename Column":
+            return self.rename_column(df_copy)
+        if op == "Add Column":
+            return self.add_column(df_copy)
+        if op == "Select Columns":
+            return self.select_columns(df_copy)
+        if op == "Head":
+            return self.head(df_copy)
+        if op == "Tail":
+            return self.tail(df_copy)
+        if op == "Replace Value":
+            return self.replace_values(df_copy)
+        if op == "Drop Duplicates":
+            return self.drop_duplicates(df_copy)
+        msg = f"Unsupported operation: {op}"
+        logger.error(msg)
+        raise ValueError(msg)
+
+    def filter_rows_by_value(self, df: DataFrame) -> DataFrame:
+        column = df[self.column_name]
+        filter_value = self.filter_value
+
+        # Handle regular DropdownInput format (just a string value)
+        operator = getattr(self, "filter_operator", "equals")  # Default to equals for backward compatibility
+
+        if operator == "equals":
+            mask = column == filter_value
+        elif operator == "not equals":
+            mask = column != filter_value
+        elif operator == "contains":
+            mask = column.astype(str).str.contains(str(filter_value), na=False)
+        elif operator == "not contains":
+            mask = ~column.astype(str).str.contains(str(filter_value), na=False)
+        elif operator == "starts with":
+            mask = column.astype(str).str.startswith(str(filter_value), na=False)
+        elif operator == "ends with":
+            mask = column.astype(str).str.endswith(str(filter_value), na=False)
+        elif operator == "greater than":
+            try:
+                # Try to convert filter_value to numeric for comparison
+                numeric_value = pd.to_numeric(filter_value)
+                mask = column > numeric_value
+            except (ValueError, TypeError):
+                # If conversion fails, compare as strings
+                mask = column.astype(str) > str(filter_value)
+        elif operator == "less than":
+            try:
+                # Try to convert filter_value to numeric for comparison
+                numeric_value = pd.to_numeric(filter_value)
+                mask = column < numeric_value
+            except (ValueError, TypeError):
+                # If conversion fails, compare as strings
+                mask = column.astype(str) < str(filter_value)
+        else:
+            mask = column == filter_value  # Fallback to equals
+
+        return DataFrame(df[mask])
+
+    def sort_by_column(self, df: DataFrame) -> DataFrame:
+        return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))
+
+    def drop_column(self, df: DataFrame) -> DataFrame:
+        return DataFrame(df.drop(columns=[self.column_name]))
+
+    def rename_column(self, df: DataFrame) -> DataFrame:
+        return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))
+
+    def add_column(self, df: DataFrame) -> DataFrame:
+        df[self.new_column_name] = [self.new_column_value] * len(df)
+        return DataFrame(df)
+
+    def select_columns(self, df: DataFrame) -> DataFrame:
+        columns = [col.strip() for col in self.columns_to_select]
+        return DataFrame(df[columns])
+
+    def head(self, df: DataFrame) -> DataFrame:
+        return DataFrame(df.head(self.num_rows))
+
+    def tail(self, df: DataFrame) -> DataFrame:
+        return DataFrame(df.tail(self.num_rows))
+
+    def replace_values(self, df: DataFrame) -> DataFrame:
+        df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)
+        return DataFrame(df)
+
+    def drop_duplicates(self, df: DataFrame) -> DataFrame:
+        return DataFrame(df.drop_duplicates(subset=self.column_name))

lfx/components/processing/extract_key.py

@@ -0,0 +1,53 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, Output, StrInput
+from lfx.schema.data import Data
+
+
+class ExtractDataKeyComponent(Component):
+    display_name = "Extract Key"
+    description = (
+        "Extract a specific key from a Data object or a list of "
+        "Data objects and return the extracted value(s) as Data object(s)."
+    )
+    icon = "key"
+    name = "ExtractaKey"
+    legacy = True
+
+    inputs = [
+        DataInput(
+            name="data_input",
+            display_name="Data Input",
+            info="The Data object or list of Data objects to extract the key from.",
+        ),
+        StrInput(
+            name="key",
+            display_name="Key to Extract",
+            info="The key in the Data object(s) to extract.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Extracted Data", name="extracted_data", method="extract_key"),
+    ]
+
+    def extract_key(self) -> Data | list[Data]:
+        key = self.key
+
+        if isinstance(self.data_input, list):
+            result = []
+            for item in self.data_input:
+                if isinstance(item, Data) and key in item.data:
+                    extracted_value = item.data[key]
+                    result.append(Data(data={key: extracted_value}))
+            self.status = result
+            return result
+        if isinstance(self.data_input, Data):
+            if key in self.data_input.data:
+                extracted_value = self.data_input.data[key]
+                result = Data(data={key: extracted_value})
+                self.status = result
+                return result
+            self.status = f"Key '{key}' not found in Data object."
+            return Data(data={"error": f"Key '{key}' not found in Data object."})
+        self.status = "Invalid input. Expected Data object or list of Data objects."
+        return Data(data={"error": "Invalid input. Expected Data object or list of Data objects."})

lfx/components/processing/filter_data.py

@@ -0,0 +1,42 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, MessageTextInput, Output
+from lfx.schema.data import Data
+
+
+class FilterDataComponent(Component):
+    display_name = "Filter Data"
+    description = "Filters a Data object based on a list of keys."
+    icon = "filter"
+    beta = True
+    name = "FilterData"
+    legacy = True
+
+    inputs = [
+        DataInput(
+            name="data",
+            display_name="Data",
+            info="Data object to filter.",
+        ),
+        MessageTextInput(
+            name="filter_criteria",
+            display_name="Filter Criteria",
+            info="List of keys to filter by.",
+            is_list=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Filtered Data", name="filtered_data", method="filter_data"),
+    ]
+
+    def filter_data(self) -> Data:
+        filter_criteria: list[str] = self.filter_criteria
+        data = self.data.data if isinstance(self.data, Data) else {}
+
+        # Filter the data
+        filtered = {key: value for key, value in data.items() if key in filter_criteria}
+
+        # Create a new Data object with the filtered data
+        filtered_data = Data(data=filtered)
+        self.status = filtered_data
+        return filtered_data

lfx/components/processing/filter_data_values.py

@@ -0,0 +1,88 @@
+from typing import Any
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, DropdownInput, MessageTextInput, Output
+from lfx.schema.data import Data
+
+
+class DataFilterComponent(Component):
+    display_name = "Filter Values"
+    description = (
+        "Filter a list of data items based on a specified key, filter value,"
+        " and comparison operator. Check advanced options to select match comparision."
+    )
+    icon = "filter"
+    beta = True
+    name = "FilterDataValues"
+    legacy = True
+
+    inputs = [
+        DataInput(name="input_data", display_name="Input Data", info="The list of data items to filter.", is_list=True),
+        MessageTextInput(
+            name="filter_key",
+            display_name="Filter Key",
+            info="The key to filter on (e.g., 'route').",
+            value="route",
+            input_types=["Data"],
+        ),
+        MessageTextInput(
+            name="filter_value",
+            display_name="Filter Value",
+            info="The value to filter by (e.g., 'CMIP').",
+            value="CMIP",
+            input_types=["Data"],
+        ),
+        DropdownInput(
+            name="operator",
+            display_name="Comparison Operator",
+            options=["equals", "not equals", "contains", "starts with", "ends with"],
+            info="The operator to apply for comparing the values.",
+            value="equals",
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Filtered Data", name="filtered_data", method="filter_data"),
+    ]
+
+    def compare_values(self, item_value: Any, filter_value: str, operator: str) -> bool:
+        if operator == "equals":
+            return str(item_value) == filter_value
+        if operator == "not equals":
+            return str(item_value) != filter_value
+        if operator == "contains":
+            return filter_value in str(item_value)
+        if operator == "starts with":
+            return str(item_value).startswith(filter_value)
+        if operator == "ends with":
+            return str(item_value).endswith(filter_value)
+        return False
+
+    def filter_data(self) -> list[Data]:
+        # Extract inputs
+        input_data: list[Data] = self.input_data
+        filter_key: str = self.filter_key.text
+        filter_value: str = self.filter_value.text
+        operator: str = self.operator
+
+        # Validate inputs
+        if not input_data:
+            self.status = "Input data is empty."
+            return []
+
+        if not filter_key or not filter_value:
+            self.status = "Filter key or value is missing."
+            return input_data
+
+        # Filter the data
+        filtered_data = []
+        for item in input_data:
+            if isinstance(item.data, dict) and filter_key in item.data:
+                if self.compare_values(item.data[filter_key], filter_value, operator):
+                    filtered_data.append(item)
+            else:
+                self.status = f"Warning: Some items don't have the key '{filter_key}' or are not dictionaries."
+
+        self.status = filtered_data
+        return filtered_data

lfx/components/processing/json_cleaner.py

@@ -0,0 +1,103 @@
+import json
+import unicodedata
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import BoolInput, MessageTextInput
+from lfx.schema.message import Message
+from lfx.template.field.base import Output
+
+
+class JSONCleaner(Component):
+    icon = "braces"
+    display_name = "JSON Cleaner"
+    description = (
+        "Cleans the messy and sometimes incorrect JSON strings produced by LLMs "
+        "so that they are fully compliant with the JSON spec."
+    )
+    legacy = True
+    inputs = [
+        MessageTextInput(
+            name="json_str", display_name="JSON String", info="The JSON string to be cleaned.", required=True
+        ),
+        BoolInput(
+            name="remove_control_chars",
+            display_name="Remove Control Characters",
+            info="Remove control characters from the JSON string.",
+            required=False,
+        ),
+        BoolInput(
+            name="normalize_unicode",
+            display_name="Normalize Unicode",
+            info="Normalize Unicode characters in the JSON string.",
+            required=False,
+        ),
+        BoolInput(
+            name="validate_json",
+            display_name="Validate JSON",
+            info="Validate the JSON string to ensure it is well-formed.",
+            required=False,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Cleaned JSON String", name="output", method="clean_json"),
+    ]
+
+    def clean_json(self) -> Message:
+        try:
+            from json_repair import repair_json
+        except ImportError as e:
+            msg = "Could not import the json_repair package. Please install it with `pip install json_repair`."
+            raise ImportError(msg) from e
+
+        """Clean the input JSON string based on provided options and return the cleaned JSON string."""
+        json_str = self.json_str
+        remove_control_chars = self.remove_control_chars
+        normalize_unicode = self.normalize_unicode
+        validate_json = self.validate_json
+
+        start = json_str.find("{")
+        end = json_str.rfind("}")
+        if start == -1 or end == -1:
+            msg = "Invalid JSON string: Missing '{' or '}'"
+            raise ValueError(msg)
+        try:
+            json_str = json_str[start : end + 1]
+
+            if remove_control_chars:
+                json_str = self._remove_control_characters(json_str)
+            if normalize_unicode:
+                json_str = self._normalize_unicode(json_str)
+            if validate_json:
+                json_str = self._validate_json(json_str)
+
+            cleaned_json_str = repair_json(json_str)
+            result = str(cleaned_json_str)
+
+            self.status = result
+            return Message(text=result)
+        except Exception as e:
+            msg = f"Error cleaning JSON string: {e}"
+            raise ValueError(msg) from e
+
+    def _remove_control_characters(self, s: str) -> str:
+        """Remove control characters from the string."""
+        return s.translate(self.translation_table)
+
+    def _normalize_unicode(self, s: str) -> str:
+        """Normalize Unicode characters in the string."""
+        return unicodedata.normalize("NFC", s)
+
+    def _validate_json(self, s: str) -> str:
+        """Validate the JSON string."""
+        try:
+            json.loads(s)
+        except json.JSONDecodeError as e:
+            msg = f"Invalid JSON string: {e}"
+            raise ValueError(msg) from e
+        return s
+
+    def __init__(self, *args, **kwargs):
+        # Create a translation table that maps control characters to None
+        super().__init__(*args, **kwargs)
+        self.translation_table = str.maketrans("", "", "".join(chr(i) for i in range(32)) + chr(127))