Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_core/flowfile/manage/io_flowfile.py

@@ -1,14 +1,13 @@
-from flowfile_core.schemas import schemas, input_schema
-from typing import List, Tuple
-from flowfile_core.flowfile.manage.compatibility_enhancements import ensure_compatibility, load_flowfile_pickle
-from flowfile_core.flowfile.flow_graph import FlowGraph
+import json
 from pathlib import Path
+
 from flowfile_core.configs.node_store import CUSTOM_NODE_STORE
-from flowfile_core.configs.settings import IS_RUNNING_IN_DOCKER
-import json
+from flowfile_core.configs.settings import is_docker_mode
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.flowfile.manage.compatibility_enhancements import ensure_compatibility, load_flowfile_pickle
+from flowfile_core.schemas import input_schema, schemas
 from shared.storage_config import storage
 
-
 try:
     import yaml
 except ImportError:
@@ -20,7 +19,7 @@ def _validate_flow_path(flow_path: Path) -> Path:
     resolved = flow_path.resolve()
 
     # Check extension
-    allowed_extensions = {'.yaml', '.yml', '.json', '.flowfile'}
+    allowed_extensions = {".yaml", ".yml", ".json", ".flowfile"}
     if resolved.suffix.lower() not in allowed_extensions:
         raise ValueError(f"Unsupported file extension: {resolved.suffix}")
 
@@ -30,16 +29,13 @@ def _validate_flow_path(flow_path: Path) -> Path:
 
     # Allow paths within known safe directories
 
-    if IS_RUNNING_IN_DOCKER:
+    if is_docker_mode():
         safe_directories = [
             storage.flows_directory,
             storage.uploads_directory,
             storage.temp_directory_for_flows,
         ]
-        is_safe = any(
-            resolved.is_relative_to(safe_dir)
-            for safe_dir in safe_directories
-        )
+        is_safe = any(resolved.is_relative_to(safe_dir) for safe_dir in safe_directories)
     else:
         is_safe = True
 
@@ -52,7 +48,7 @@ def _validate_flow_path(flow_path: Path) -> Path:
     return resolved
 
 
-def _derive_connections_from_nodes(nodes: List[schemas.FlowfileNode]) -> List[Tuple[int, int]]:
+def _derive_connections_from_nodes(nodes: list[schemas.FlowfileNode]) -> list[tuple[int, int]]:
     """Derive node connections from the outputs stored in each node."""
     connections = []
     for node in nodes:
@@ -63,7 +59,7 @@ def _derive_connections_from_nodes(nodes: List[schemas.FlowfileNode]) -> List[Tu
 
 
 def determine_insertion_order(node_storage: schemas.FlowInformation):
-    ingest_order: List[int] = []
+    ingest_order: list[int] = []
     ingest_order_set: set[int] = set()
     all_nodes = set(node_storage.data.keys())
 
@@ -78,10 +74,11 @@ def determine_insertion_order(node_storage: schemas.FlowInformation):
             return
         output_ids = current_node.outputs
         main_input_ids = current_node.input_ids if current_node.input_ids else []
-        input_ids = [n for n in [current_node.left_input_id,
-                                 current_node.right_input_id] + main_input_ids if (n is not None
-                                                                                   and n not in
-                                                                                   ingest_order_set)]
+        input_ids = [
+            n
+            for n in [current_node.left_input_id, current_node.right_input_id] + main_input_ids
+            if (n is not None and n not in ingest_order_set)
+        ]
         if len(input_ids) > 0:
             for input_id in input_ids:
                 new_node = node_storage.data.get(input_id)
@@ -123,7 +120,7 @@ def _load_flowfile_yaml(flow_path: Path) -> schemas.FlowInformation:
     if yaml is None:
         raise ImportError("PyYAML is required for YAML files. Install with: pip install pyyaml")
     flow_path = _validate_flow_path(flow_path)
-    with open(flow_path, 'r', encoding='utf-8') as f:
+    with open(flow_path, encoding="utf-8") as f:
         data = yaml.safe_load(f)
     # Load as FlowfileData first (handles setting_input validation via node type)
     flowfile_data = schemas.FlowfileData.model_validate(data)
@@ -142,7 +139,7 @@ def _load_flowfile_json(flow_path: Path) -> schemas.FlowInformation:
         FlowInformation object
     """
     flow_path = _validate_flow_path(flow_path)
-    with open(flow_path, 'r', encoding='utf-8') as f:
+    with open(flow_path, encoding="utf-8") as f:
         data = json.load(f)
 
     # Load as FlowfileData first (handles setting_input validation via node type)
@@ -168,40 +165,42 @@ def _flowfile_data_to_flow_information(flowfile_data: schemas.FlowfileData) -> s
             is_user_defined = model_class == input_schema.UserDefinedNode
 
             # Inject fields that were excluded during serialization
-            setting_data = node.setting_input if isinstance(node.setting_input, dict) else node.setting_input.model_dump()
-            setting_data['flow_id'] = flowfile_data.flowfile_id
-            setting_data['node_id'] = node.id
-            setting_data['pos_x'] = float(node.x_position or 0)
-            setting_data['pos_y'] = float(node.y_position or 0)
-            setting_data['description'] = node.description or ''
-            setting_data['is_setup'] = True
+            setting_data = (
+                node.setting_input if isinstance(node.setting_input, dict) else node.setting_input.model_dump()
+            )
+            setting_data["flow_id"] = flowfile_data.flowfile_id
+            setting_data["node_id"] = node.id
+            setting_data["pos_x"] = float(node.x_position or 0)
+            setting_data["pos_y"] = float(node.y_position or 0)
+            setting_data["description"] = node.description or ""
+            setting_data["is_setup"] = True
 
             if is_user_defined:
-                setting_data['is_user_defined'] = True
+                setting_data["is_user_defined"] = True
                 depending_ids = list(node.input_ids or [])
                 if node.left_input_id:
                     depending_ids.append(node.left_input_id)
                 if node.right_input_id:
                     depending_ids.append(node.right_input_id)
-                setting_data['depending_on_ids'] = depending_ids
+                setting_data["depending_on_ids"] = depending_ids
             else:
-                if 'depending_on_id' in model_class.model_fields:
-                    setting_data['depending_on_id'] = node.input_ids[0] if node.input_ids else -1
-                if 'depending_on_ids' in model_class.model_fields:
+                if "depending_on_id" in model_class.model_fields:
+                    setting_data["depending_on_id"] = node.input_ids[0] if node.input_ids else -1
+                if "depending_on_ids" in model_class.model_fields:
                     depending_ids = list(node.input_ids or [])
                     if node.left_input_id:
                         depending_ids.append(node.left_input_id)
                     if node.right_input_id:
                         depending_ids.append(node.right_input_id)
-                    setting_data['depending_on_ids'] = depending_ids
+                    setting_data["depending_on_ids"] = depending_ids
 
-                if node.type == 'output' and 'output_settings' in setting_data:
-                    output_settings = setting_data['output_settings']
-                    file_type = output_settings.get('file_type', None)
+                if node.type == "output" and "output_settings" in setting_data:
+                    output_settings = setting_data["output_settings"]
+                    file_type = output_settings.get("file_type", None)
                     if file_type is None:
                         raise ValueError("Output node's output_settings must include 'file_type'")
-                    if 'table_settings' not in output_settings:
-                        output_settings['table_settings'] = {"file_type": file_type}
+                    if "table_settings" not in output_settings:
+                        output_settings["table_settings"] = {"file_type": file_type}
 
             setting_input = model_class.model_validate(setting_data)
 
@@ -243,6 +242,7 @@ def _flowfile_data_to_flow_information(flowfile_data: schemas.FlowfileData) -> s
         node_connections=connections,
     )
 
+
 def _load_flow_storage(flow_path: Path) -> schemas.FlowInformation:
     """
     Load flow storage from any supported format.
@@ -260,21 +260,20 @@ def _load_flow_storage(flow_path: Path) -> schemas.FlowInformation:
     """
     flow_path = _validate_flow_path(flow_path)
     suffix = flow_path.suffix.lower()
-    if suffix == '.flowfile':
+    if suffix == ".flowfile":
         try:
             flow_storage_obj = load_flowfile_pickle(str(flow_path))
             ensure_compatibility(flow_storage_obj, str(flow_path))
             return flow_storage_obj
         except Exception as e:
             raise ValueError(
-                f"Failed to open legacy .flowfile: {e}\n\n"
-                f"Try migrating: migrate_flowfile('{flow_path}')"
+                f"Failed to open legacy .flowfile: {e}\n\n" f"Try migrating: migrate_flowfile('{flow_path}')"
             ) from e
 
-    elif suffix in ('.yaml', '.yml'):
+    elif suffix in (".yaml", ".yml"):
         return _load_flowfile_yaml(flow_path)
 
-    elif suffix == '.json':
+    elif suffix == ".json":
         return _load_flowfile_json(flow_path)
     else:
         raise ValueError(f"Unsupported file format: {suffix}")
@@ -304,9 +303,8 @@ def open_flow(flow_path: Path) -> FlowGraph:
 
     # Determine node insertion order
     ingestion_order = determine_insertion_order(flow_storage_obj)
-
-    # Create new FlowGraph
     new_flow = FlowGraph(name=flow_storage_obj.flow_name, flow_settings=flow_storage_obj.flow_settings)
+    # Create new FlowGraph
 
     # First pass: add node promises
     for node_id in ingestion_order:
@@ -316,9 +314,9 @@ def open_flow(flow_path: Path) -> FlowGraph:
             node_id=node_info.id,
             pos_x=node_info.x_position,
             pos_y=node_info.y_position,
-            node_type=node_info.type
+            node_type=node_info.type,
         )
-        if hasattr(node_info.setting_input, 'cache_results'):
+        if hasattr(node_info.setting_input, "cache_results"):
             node_promise.cache_results = node_info.setting_input.cache_results
         new_flow.add_node_promise(node_promise)
 
@@ -331,15 +329,14 @@ def open_flow(flow_path: Path) -> FlowGraph:
                 user_defined_node_class = CUSTOM_NODE_STORE[node_info.type]
                 new_flow.add_user_defined_node(
                     custom_node=user_defined_node_class.from_settings(node_info.setting_input.settings),
-                    user_defined_node_settings=node_info.setting_input
+                    user_defined_node_settings=node_info.setting_input,
                 )
             else:
-                getattr(new_flow, 'add_' + node_info.type)(node_info.setting_input)
+                getattr(new_flow, "add_" + node_info.type)(node_info.setting_input)
 
         # Setup connections
         from_node = new_flow.get_node(node_id)
-        for output_node_id in (node_info.outputs or []):
-
+        for output_node_id in node_info.outputs or []:
             to_node = new_flow.get_node(output_node_id)
             if to_node is not None:
                 output_node_obj = flow_storage_obj.data[output_node_id]
@@ -352,17 +349,17 @@ def open_flow(flow_path: Path) -> FlowGraph:
                 is_main_input = node_id in (output_node_obj.input_ids or [])
 
                 if is_left_input:
-                    insert_type = 'left'
+                    insert_type = "left"
                 elif is_right_input:
-                    insert_type = 'right'
+                    insert_type = "right"
                 elif is_main_input:
-                    insert_type = 'main'
+                    insert_type = "main"
                 else:
                     continue
                 to_node.add_node_connection(from_node, insert_type)
             else:
                 from_node.delete_lead_to_node(output_node_id)
-                if not (from_node.node_id, output_node_id) in flow_storage_obj.node_connections:
+                if (from_node.node_id, output_node_id) not in flow_storage_obj.node_connections:
                     continue
                 flow_storage_obj.node_connections.pop(
                     flow_storage_obj.node_connections.index((from_node.node_id, output_node_id))
@@ -380,7 +377,7 @@ def open_flow(flow_path: Path) -> FlowGraph:
     return new_flow
 
 
-def test_if_circular_connection(connection: Tuple[int, int], flow: FlowGraph):
+def test_if_circular_connection(connection: tuple[int, int], flow: FlowGraph):
     to_node = flow.get_node(connection[1])
     leads_to_nodes_queue = [n for n in to_node.leads_to_nodes]
     circular_connection: bool = False

flowfile_core/flowfile/node_designer/__init__.py

@@ -8,40 +8,42 @@ custom nodes, define their UI, and implement their data processing logic.
 """
 
 # Import the core base class for creating a new node
+# Import the main `Types` object for filtering in ColumnSelector
+from flowfile_core.types import Types
+
 from .custom_node import CustomNodeBase, NodeSettings
 
 # Import all UI components so they can be used directly
 from .ui_components import (
+    AvailableSecrets,
+    ColumnSelector,
+    IncomingColumns,
+    MultiSelect,
+    NumericInput,
+    SecretSelector,
     Section,
+    SingleSelect,
+    SliderInput,
     TextInput,
-    NumericInput,
     ToggleSwitch,
-    SingleSelect,
-    MultiSelect,
-    ColumnSelector,
-    IncomingColumns,  # Important marker class for dynamic dropdowns
 )
 
-# Import the main `Types` object for filtering in ColumnSelector
-from flowfile_core.types import Types
-
-
 # Define the public API of this package
 __all__ = [
     # Core Node Class
     "CustomNodeBase",
-
     # UI Components & Layout
     "Section",
     "TextInput",
     "NumericInput",
+    "SliderInput",
     "ToggleSwitch",
     "SingleSelect",
     "MultiSelect",
     "NodeSettings",
     "ColumnSelector",
     "IncomingColumns",
-
-    # Data Type Filtering
+    "AvailableSecrets",
+    "SecretSelector",
     "Types",
-]
+]

flowfile_core/flowfile/node_designer/_type_registry.py

@@ -5,18 +5,20 @@ This module should not be imported directly by users.
 """
 
 from dataclasses import dataclass
-from typing import Type, List, Dict, Set, Any
+from typing import Any
+
 import polars as pl
 
 # Import public types
-from flowfile_core.types import TypeGroup, DataType
+from flowfile_core.types import DataType, TypeGroup
 
 
 @dataclass(frozen=True)
 class TypeMapping:
     """Internal mapping between type representations."""
+
     data_type: DataType
-    polars_type: Type[pl.DataType]
+    polars_type: type[pl.DataType]
     type_group: TypeGroup
     aliases: tuple[str, ...] = ()
 
@@ -28,42 +30,32 @@ class TypeRegistry:
     """
 
     def __init__(self):
-        self._mappings: List[TypeMapping] = [
+        self._mappings: list[TypeMapping] = [
             # Numeric types
             TypeMapping(DataType.Int8, pl.Int8, TypeGroup.Numeric, ("i8",)),
             TypeMapping(DataType.Int16, pl.Int16, TypeGroup.Numeric, ("i16",)),
             TypeMapping(DataType.Int32, pl.Int32, TypeGroup.Numeric, ("i32", "int32")),
-            TypeMapping(DataType.Int64, pl.Int64, TypeGroup.Numeric,
-                        ("i64", "int64", "int", "integer", "bigint")),
+            TypeMapping(DataType.Int64, pl.Int64, TypeGroup.Numeric, ("i64", "int64", "int", "integer", "bigint")),
             TypeMapping(DataType.UInt8, pl.UInt8, TypeGroup.Numeric, ("u8",)),
             TypeMapping(DataType.UInt16, pl.UInt16, TypeGroup.Numeric, ("u16",)),
             TypeMapping(DataType.UInt32, pl.UInt32, TypeGroup.Numeric, ("u32", "uint32")),
             TypeMapping(DataType.UInt64, pl.UInt64, TypeGroup.Numeric, ("u64", "uint64")),
             TypeMapping(DataType.Float32, pl.Float32, TypeGroup.Numeric, ("f32", "float32")),
-            TypeMapping(DataType.Float64, pl.Float64, TypeGroup.Numeric,
-                        ("f64", "float64", "float", "double")),
-            TypeMapping(DataType.Decimal, pl.Decimal, TypeGroup.Numeric,
-                        ("decimal", "numeric", "dec")),
-
+            TypeMapping(DataType.Float64, pl.Float64, TypeGroup.Numeric, ("f64", "float64", "float", "double")),
+            TypeMapping(DataType.Decimal, pl.Decimal, TypeGroup.Numeric, ("decimal", "dec")),
             # String types
-            TypeMapping(DataType.String, pl.String, TypeGroup.String,
-                        ("str", "string", "utf8", "varchar", "text")),
-            TypeMapping(DataType.Categorical, pl.Categorical, TypeGroup.String,
-                        ("cat", "categorical", "enum", "factor")),
-
+            TypeMapping(DataType.String, pl.String, TypeGroup.String, ("str", "string", "utf8", "varchar", "text")),
+            TypeMapping(
+                DataType.Categorical, pl.Categorical, TypeGroup.String, ("cat", "categorical", "enum", "factor")
+            ),
             # Date types
             TypeMapping(DataType.Date, pl.Date, TypeGroup.Date, ("date",)),
-            TypeMapping(DataType.Datetime, pl.Datetime, TypeGroup.Date,
-                        ("datetime", "timestamp")),
+            TypeMapping(DataType.Datetime, pl.Datetime, TypeGroup.Date, ("datetime", "timestamp")),
             TypeMapping(DataType.Time, pl.Time, TypeGroup.Date, ("time",)),
-            TypeMapping(DataType.Duration, pl.Duration, TypeGroup.Date,
-                        ("duration", "timedelta")),
-
+            TypeMapping(DataType.Duration, pl.Duration, TypeGroup.Date, ("duration", "timedelta")),
             # Other types
-            TypeMapping(DataType.Boolean, pl.Boolean, TypeGroup.Boolean,
-                        ("bool", "boolean")),
-            TypeMapping(DataType.Binary, pl.Binary, TypeGroup.Binary,
-                        ("binary", "bytes", "bytea")),
+            TypeMapping(DataType.Boolean, pl.Boolean, TypeGroup.Boolean, ("bool", "boolean")),
+            TypeMapping(DataType.Binary, pl.Binary, TypeGroup.Binary, ("binary", "bytes", "bytea")),
             TypeMapping(DataType.List, pl.List, TypeGroup.Complex, ("list", "array")),
             TypeMapping(DataType.Struct, pl.Struct, TypeGroup.Complex, ("struct", "object")),
             TypeMapping(DataType.Array, pl.Array, TypeGroup.Complex, ("fixed_array",)),
@@ -73,10 +65,10 @@ class TypeRegistry:
 
     def _build_indices(self):
         """Build lookup indices for fast access."""
-        self._by_data_type: Dict[DataType, TypeMapping] = {}
-        self._by_polars_type: Dict[Type[pl.DataType], TypeMapping] = {}
-        self._by_alias: Dict[str, TypeMapping] = {}
-        self._by_group: Dict[TypeGroup, List[TypeMapping]] = {g: [] for g in TypeGroup}
+        self._by_data_type: dict[DataType, TypeMapping] = {}
+        self._by_polars_type: dict[type[pl.DataType], TypeMapping] = {}
+        self._by_alias: dict[str, TypeMapping] = {}
+        self._by_group: dict[TypeGroup, list[TypeMapping]] = {g: [] for g in TypeGroup}
 
         for mapping in self._mappings:
             self._by_data_type[mapping.data_type] = mapping
@@ -96,12 +88,13 @@ class TypeRegistry:
             # Register "pl.TypeName" format
             self._by_alias[f"pl.{mapping.polars_type.__name__}".lower()] = mapping
 
-    def normalize(self, type_spec: Any) -> Set[DataType]:
+    def normalize(self, type_spec: Any) -> set[DataType]:
         """
         Normalize any type specification to a set of DataType enums.
         This is the main internal API for type resolution.
         """
         # Handle special case: All types
+
         if type_spec == TypeGroup.All or type_spec == "ALL":
             return set(self._by_data_type.keys())
 
@@ -121,7 +114,7 @@ class TypeRegistry:
 
         # Handle Polars type instance
         if isinstance(type_spec, pl.DataType):
-            base_type = type_spec.base_type() if hasattr(type_spec, 'base_type') else type(type_spec)
+            base_type = type_spec.base_type() if hasattr(type_spec, "base_type") else type(type_spec)
             mapping = self._by_polars_type.get(base_type)
             if mapping:
                 return {mapping.data_type}
@@ -129,6 +122,10 @@ class TypeRegistry:
         # Handle string aliases
         if isinstance(type_spec, str):
             type_spec_lower = type_spec.lower()
+            group: TypeGroup
+            for group in TypeGroup:
+                if group.lower() == type_spec_lower:
+                    return {m.data_type for m in (self._by_group.get(group) or [])}
 
             # Try TypeGroup name
             try:
@@ -152,14 +149,14 @@ class TypeRegistry:
         # Default to empty set if unrecognized
         return set()
 
-    def normalize_list(self, type_specs: List[Any]) -> Set[DataType]:
+    def normalize_list(self, type_specs: list[Any]) -> set[DataType]:
         """Normalize a list of type specifications."""
         result = set()
         for spec in type_specs:
             result.update(self.normalize(spec))
         return result
 
-    def get_polars_types(self, data_types: Set[DataType]) -> Set[Type[pl.DataType]]:
+    def get_polars_types(self, data_types: set[DataType]) -> set[type[pl.DataType]]:
         """Convert a set of DataType enums to Polars types."""
         result = set()
         for dt in data_types:
@@ -168,7 +165,7 @@ class TypeRegistry:
                 result.add(mapping.polars_type)
         return result
 
-    def get_polars_type(self, data_type: DataType) -> Type[pl.DataType]:
+    def get_polars_type(self, data_type: DataType) -> type[pl.DataType]:
         """Get the Polars type for a single DataType."""
         mapping = self._by_data_type.get(data_type)
         return mapping.polars_type if mapping else pl.String  # Default fallback
@@ -179,19 +176,19 @@ _registry = TypeRegistry()
 
 
 # Internal API functions (not for public use)
-def normalize_type_spec(type_spec: Any) -> Set[DataType]:
+def normalize_type_spec(type_spec: Any) -> set[DataType]:
     """Internal function to normalize type specifications."""
     if isinstance(type_spec, list):
         return _registry.normalize_list(type_spec)
     return _registry.normalize(type_spec)
 
 
-def get_polars_types(data_types: Set[DataType]) -> Set[Type[pl.DataType]]:
+def get_polars_types(data_types: set[DataType]) -> set[type[pl.DataType]]:
     """Internal function to get Polars types."""
     return _registry.get_polars_types(data_types)
 
 
-def check_column_type(column_dtype: pl.DataType, accepted_types: Set[DataType]) -> bool:
+def check_column_type(column_dtype: pl.DataType, accepted_types: set[DataType]) -> bool:
     """Check if a column's dtype matches the accepted types."""
     normalized = _registry.normalize(column_dtype)
     return bool(normalized & accepted_types)