Flowfile 0.4.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_worker/spawner.py

@@ -1,15 +1,14 @@
-from flowfile_worker import status_dict
-from time import sleep
 import gc
-from typing import List, Tuple
 from multiprocessing import Process, Queue
+from time import sleep
+
+from flowfile_worker import funcs, models, mp_context, status_dict, status_dict_lock
 from flowfile_worker.process_manager import ProcessManager
-from flowfile_worker import models, mp_context, funcs, status_dict_lock
 
 # Initialize ProcessManager
 process_manager = ProcessManager()
 
-flowfile_node_id_type = int|str
+flowfile_node_id_type = int | str
 
 
 def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_message: mp_context.Array, q: Queue):
@@ -48,7 +47,7 @@ def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_mess
                 with status_dict_lock:
                     status_dict[task_id].status = "Error"
                     with error_message.get_lock():
-                        status_dict[task_id].error_message = error_message.value.decode().rstrip('\x00')
+                        status_dict[task_id].error_message = error_message.value.decode().rstrip("\x00")
                 break
 
         p.join()
@@ -72,11 +71,15 @@ def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_mess
         gc.collect()
 
 
-def start_process(polars_serializable_object: bytes, task_id: str,
-                  operation: models.OperationType,
-                  file_ref: str, flowfile_flow_id: int,
-                  flowfile_node_id: flowfile_node_id_type,
-                  kwargs: dict = None) -> None:
+def start_process(
+    polars_serializable_object: bytes,
+    task_id: str,
+    operation: models.OperationType,
+    file_ref: str,
+    flowfile_flow_id: int,
+    flowfile_node_id: flowfile_node_id_type,
+    kwargs: dict = None,
+) -> None:
     """
     Starts a new process for handling Polars dataframe operations.
 
@@ -97,24 +100,31 @@ def start_process(polars_serializable_object: bytes, task_id: str,
     if kwargs is None:
         kwargs = {}
     process_task = getattr(funcs, operation)
-    kwargs['polars_serializable_object'] = polars_serializable_object
-    kwargs['progress'] = mp_context.Value('i', 0)
-    kwargs['error_message'] = mp_context.Array('c', 1024)
-    kwargs['queue'] = Queue(maxsize=1)
-    kwargs['file_path'] = file_ref
-    kwargs['flowfile_flow_id'] = flowfile_flow_id
-    kwargs['flowfile_node_id'] = flowfile_node_id
+    kwargs["polars_serializable_object"] = polars_serializable_object
+    kwargs["progress"] = mp_context.Value("i", 0)
+    kwargs["error_message"] = mp_context.Array("c", 1024)
+    kwargs["queue"] = Queue(maxsize=1)
+    kwargs["file_path"] = file_ref
+    kwargs["flowfile_flow_id"] = flowfile_flow_id
+    kwargs["flowfile_node_id"] = flowfile_node_id
 
     p: Process = mp_context.Process(target=process_task, kwargs=kwargs)
     p.start()
 
     process_manager.add_process(task_id, p)
-    handle_task(task_id=task_id, p=p, progress=kwargs['progress'], error_message=kwargs['error_message'], q=kwargs['queue'])
-
-
-def start_generic_process(func_ref: callable, task_id: str,
-                          file_ref: str, flowfile_flow_id: int,
-                          flowfile_node_id: flowfile_node_id_type, kwargs: dict = None) -> None:
+    handle_task(
+        task_id=task_id, p=p, progress=kwargs["progress"], error_message=kwargs["error_message"], q=kwargs["queue"]
+    )
+
+
+def start_generic_process(
+    func_ref: callable,
+    task_id: str,
+    file_ref: str,
+    flowfile_flow_id: int,
+    flowfile_node_id: flowfile_node_id_type,
+    kwargs: dict = None,
+) -> None:
     """
     Starts a new process for handling generic function execution.
 
@@ -132,30 +142,33 @@ def start_generic_process(func_ref: callable, task_id: str,
         - Delegates to handle_task for process monitoring
     """
     kwargs = {} if kwargs is None else kwargs
-    kwargs['func'] = func_ref
-    kwargs['progress'] = mp_context.Value('i', 0)
-    kwargs['error_message'] = mp_context.Array('c', 1024)
-    kwargs['queue'] = Queue(maxsize=1)
-    kwargs['file_path'] = file_ref
-    kwargs['flowfile_flow_id'] = flowfile_flow_id
-    kwargs['flowfile_node_id'] = flowfile_node_id
-
-    process_task = getattr(funcs, 'generic_task')
+    kwargs["func"] = func_ref
+    kwargs["progress"] = mp_context.Value("i", 0)
+    kwargs["error_message"] = mp_context.Array("c", 1024)
+    kwargs["queue"] = Queue(maxsize=1)
+    kwargs["file_path"] = file_ref
+    kwargs["flowfile_flow_id"] = flowfile_flow_id
+    kwargs["flowfile_node_id"] = flowfile_node_id
+
+    process_task = funcs.generic_task
     p: Process = mp_context.Process(target=process_task, kwargs=kwargs)
     p.start()
 
     process_manager.add_process(task_id, p)  # Add process to process manager
-    handle_task(task_id=task_id, p=p, progress=kwargs['progress'],
-                error_message=kwargs['error_message'], q=kwargs['queue'])
-
-
-def start_fuzzy_process(left_serializable_object: bytes,
-                        right_serializable_object: bytes,
-                        file_ref: str,
-                        fuzzy_maps: List[models.FuzzyMapping],
-                        task_id: str,
-                        flowfile_flow_id: int,
-                        flowfile_node_id: flowfile_node_id_type) -> None:
+    handle_task(
+        task_id=task_id, p=p, progress=kwargs["progress"], error_message=kwargs["error_message"], q=kwargs["queue"]
+    )
+
+
+def start_fuzzy_process(
+    left_serializable_object: bytes,
+    right_serializable_object: bytes,
+    file_ref: str,
+    fuzzy_maps: list[models.FuzzyMapping],
+    task_id: str,
+    flowfile_flow_id: int,
+    flowfile_node_id: flowfile_node_id_type,
+) -> None:
     """
     Starts a new process for performing fuzzy joining operations on two datasets.
 
@@ -172,13 +185,31 @@ def start_fuzzy_process(left_serializable_object: bytes,
         - Initializes and starts a new process for fuzzy joining operation
         - Delegates to handle_task for process monitoring
     """
-    progress = mp_context.Value('i', 0)
-    error_message = mp_context.Array('c', 1024)
+    progress = mp_context.Value("i", 0)
+    error_message = mp_context.Array("c", 1024)
     q = Queue(maxsize=1)
 
-    args: Tuple[bytes, bytes, List[models.FuzzyMapping], mp_context.Array, str, mp_context.Value, Queue, int, flowfile_node_id_type] = \
-        (left_serializable_object, right_serializable_object, fuzzy_maps, error_message, file_ref, progress, q,
-         flowfile_flow_id, flowfile_node_id)
+    args: tuple[
+        bytes,
+        bytes,
+        list[models.FuzzyMapping],
+        mp_context.Array,
+        str,
+        mp_context.Value,
+        Queue,
+        int,
+        flowfile_node_id_type,
+    ] = (
+        left_serializable_object,
+        right_serializable_object,
+        fuzzy_maps,
+        error_message,
+        file_ref,
+        progress,
+        q,
+        flowfile_flow_id,
+        flowfile_node_id,
+    )
 
     p: Process = mp_context.Process(target=funcs.fuzzy_join_task, args=args)
     p.start()

flowfile_worker/utils.py

@@ -1,6 +1,7 @@
+from dataclasses import dataclass
+
 import polars as pl
 from polars.exceptions import PanicException
-from dataclasses import dataclass
 
 
 def collect_lazy_frame(lf: pl.LazyFrame) -> pl.DataFrame:
@@ -12,7 +13,7 @@ def collect_lazy_frame(lf: pl.LazyFrame) -> pl.DataFrame:
 
 @dataclass
 class CollectStreamingInfo:
-    __slots__ = 'df', 'streaming_collect_available'
+    __slots__ = "df", "streaming_collect_available"
     df: pl.DataFrame
     streaming_collect_available: bool
 

shared/__init__.py

@@ -5,11 +5,6 @@ flowfile_core, flowfile_worker, and other components without creating
 circular dependencies.
 """
 
-from .storage_config import storage, get_cache_directory, get_temp_directory, get_flows_directory
+from .storage_config import get_cache_directory, get_flows_directory, get_temp_directory, storage
 
-__all__ = [
-    'storage',
-    'get_cache_directory', 
-    'get_temp_directory',
-    'get_flows_directory'
-]
+__all__ = ["storage", "get_cache_directory", "get_temp_directory", "get_flows_directory"]

shared/storage_config.py

@@ -3,28 +3,40 @@
 Centralized storage configuration for Flowfile.
 This module can be imported by both core and worker without creating dependencies.
 """
+
 import os
 from pathlib import Path
-from typing import Optional, Literal
+from typing import Literal
+
+DirectoryOptions = Literal[
+    "temp_directory",
+    "logs_directory",
+    "system_logs_directory",
+    "database_directory",
+    "cache_directory",
+    "flows_directory",
+    "user_defined_nodes_directory",
+]
+
 
-DirectoryOptions = Literal["temp_directory", "logs_directory",
-                            "system_logs_directory", "database_directory",
-                            "cache_directory", "flows_directory", "user_defined_nodes_directory"]
+def _is_docker_mode() -> bool:
+    """Check if running in Docker mode based on FLOWFILE_MODE."""
+    return os.environ.get("FLOWFILE_MODE") == "docker"
 
 
 class FlowfileStorage:
     """Centralized storage manager for Flowfile applications."""
 
     def __init__(self):
-        self._base_dir: Optional[Path] = None
-        self._user_data_dir: Optional[Path] = None
+        self._base_dir: Path | None = None
+        self._user_data_dir: Path | None = None
         self._ensure_directories()
 
     @property
     def base_directory(self) -> Path:
         """Get the base Flowfile storage directory (for internal container communication)."""
         if self._base_dir is None:
-            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            if _is_docker_mode():
                 # In Docker, internal storage stays inside /app
                 base_path = os.environ.get("FLOWFILE_STORAGE_DIR", "/app/internal_storage")
             else:
@@ -41,7 +53,7 @@ class FlowfileStorage:
     def user_data_directory(self) -> Path:
         """Get the user data directory (completely separate from application code)."""
         if self._user_data_dir is None:
-            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            if _is_docker_mode():
                 # In Docker, user data is at /data/user (completely outside /app)
                 user_data_path = os.environ.get("FLOWFILE_USER_DATA_DIR", "/data/user")
             else:
@@ -70,7 +82,7 @@ class FlowfileStorage:
     @property
     def flows_directory(self) -> Path:
         """Directory for flow storage (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             # In Docker, flows are in separate user data area
             return self.user_data_directory / "flows"
         else:
@@ -80,7 +92,7 @@ class FlowfileStorage:
     @property
     def uploads_directory(self) -> Path:
         """Directory for user uploads (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             # In Docker, uploads are in separate user data area
             return self.user_data_directory / "uploads"
         else:
@@ -90,7 +102,7 @@ class FlowfileStorage:
     @property
     def user_defined_nodes_directory(self) -> Path:
         """Directory for user-defined custom nodes (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             return self.user_data_directory / "user_defined_nodes"
         else:
             return self.base_directory / "user_defined_nodes"
@@ -103,7 +115,7 @@ class FlowfileStorage:
     @property
     def outputs_directory(self) -> Path:
         """Directory for user outputs (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             # In Docker, outputs are in separate user data area
             return self.user_data_directory / "outputs"
         else:
@@ -184,8 +196,8 @@ class FlowfileStorage:
 
     def cleanup_directory(self, directory_option: DirectoryOptions, storage_duration_hours: int = 24) -> None:
         """Clean up any directory of the folder"""
-        import time
         import shutil
+        import time
 
         if not hasattr(self, directory_option):
             raise Exception(f"Directory does not exist in {self.base_directory}")

test_utils/postgres/commands.py

@@ -6,8 +6,9 @@ to start and stop PostgreSQL containers with sample data.
 """
 
 import argparse
-import sys
 import logging
+import sys
+
 from . import fixtures
 
 # Set up logging
@@ -106,4 +107,4 @@ if __name__ == "__main__":
         sys.exit(stop_postgres())
     else:
         print("Usage: python -m test_utils.postgres.commands [start|stop] [options]")
-        sys.exit(1)
+        sys.exit(1)

test_utils/postgres/fixtures.py

@@ -5,13 +5,13 @@ This module provides utilities to set up, manage, and tear down PostgreSQL
 containers with sample data for testing.
 """
 
-import os
-import time
 import logging
-import subprocess
+import os
 import shutil
+import subprocess
+import time
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Dict, Generator, Optional, Tuple
 
 # Configure logging
 logging.basicConfig(
@@ -208,7 +208,7 @@ def start_postgres_container(
         container_name: str = POSTGRES_CONTAINER_NAME,
         port: int = POSTGRES_PORT,
         image_tag: str = POSTGRES_IMAGE_TAG
-) -> Tuple[Optional[subprocess.Popen], bool]:
+) -> tuple[subprocess.Popen | None, bool]:
     """
     Start the PostgreSQL container with sample data.
 
@@ -225,7 +225,7 @@ def start_postgres_container(
         logger.warning("Docker not available, skipping PostgreSQL container start")
         return None, False
 
-    logger.info(f"Starting PostgreSQL container with sample data...")
+    logger.info("Starting PostgreSQL container with sample data...")
 
     # Check if container is already running
     if is_container_running(container_name):
@@ -345,12 +345,12 @@ def print_connection_info(
     print(f"Connection string: postgresql://{user}:{password}@{host}:{port}/{db}")
     print("=" * 50)
     print("\nTo stop the container, run:")
-    print(f"poetry run stop_postgres")
+    print("poetry run stop_postgres")
     print("=" * 50 + "\n")
 
 
 @contextmanager
-def managed_postgres() -> Generator[Dict[str, any], None, None]:
+def managed_postgres() -> Generator[dict[str, any], None, None]:
     """
     Context manager for PostgreSQL container management.
     Ensures proper cleanup even when tests fail.
@@ -414,4 +414,4 @@ def get_db_engine():
         return engine
     except ImportError:
         logger.error("SQLAlchemy not installed. Run: pip install sqlalchemy")
-        raise
+        raise

test_utils/s3/commands.py

@@ -43,4 +43,4 @@ def stop_minio():
     if fixtures.stop_minio_container():
         print("MinIO stopped successfully")
         return 0
-    return 1
+    return 1

test_utils/s3/data_generator.py

@@ -1,17 +1,16 @@
 
-import logging
 import io
+import logging
 import os
 
 # Third-party libraries
 import boto3
-from botocore.client import Config
 import polars as pl
 import pyarrow as pa
+from botocore.client import Config
 from deltalake import write_deltalake
 from pyiceberg.catalog import load_catalog
 
-
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -289,4 +288,4 @@ if __name__ == '__main__':
     populate_test_data(endpoint_url=MINIO_ENDPOINT_URL,
                        access_key=MINIO_ACCESS_KEY,
                        secret_key=MINIO_SECRET_KEY,
-                       bucket_name="test-bucket")
+                       bucket_name="test-bucket")

test_utils/s3/demo_data_generator.py

@@ -1,17 +1,14 @@
-import logging
 import io
+import logging
 import os
-import tempfile
-import shutil
 import random
+import tempfile
 from datetime import datetime, timedelta
 
 # Third-party libraries
 import boto3
-from botocore.client import Config
 import polars as pl
-import pyarrow as pa
-from pyarrow import parquet as pq
+from botocore.client import Config
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -152,7 +149,7 @@ def create_demo_data(endpoint_url: str, access_key: str, secret_key: str, bucket
     unique_product_ids = sales_df["product_id"].unique().to_list()
     # Create a map of product_id to unit_price from the first occurrence in sales_df
     product_price_map = sales_df.group_by("product_id").agg(pl.first("unit_price")).to_dict(as_series=False)
-    price_dict = dict(zip(product_price_map['product_id'], product_price_map['unit_price']))
+    price_dict = dict(zip(product_price_map['product_id'], product_price_map['unit_price'], strict=False))
 
     products_df = pl.DataFrame({
         "product_id": unique_product_ids,

test_utils/s3/fixtures.py

@@ -1,12 +1,14 @@
+import logging
 import os
-import time
+import shutil
 import subprocess
-import logging
+import time
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Dict, Generator
-import shutil
+
 import boto3
 from botocore.client import Config
+
 from test_utils.s3.data_generator import populate_test_data
 from test_utils.s3.demo_data_generator import create_demo_data
 
@@ -191,7 +193,7 @@ def start_minio_container() -> bool:
 
 
 @contextmanager
-def managed_minio() -> Generator[Dict[str, any], None, None]:
+def managed_minio() -> Generator[dict[str, any], None, None]:
     """Context manager for MinIO container with full connection info"""
     if not start_minio_container():
         yield {}

tools/migrate/README.md

@@ -0,0 +1,56 @@
+# Flowfile Migration Tool
+
+Migrates `.flowfile` (pickle format, used in v0.4.1 and earlier) to YAML (v0.5+).
+
+## Installation
+
+```bash
+pip install pyyaml  # Required for YAML output
+```
+
+## Usage
+
+```bash
+# Single file
+python -m tools.migrate path/to/flow.flowfile
+
+# Directory (recursive)
+python -m tools.migrate ./flows/
+
+# Specify output path
+python -m tools.migrate flow.flowfile -o /output/path/flow.yaml
+
+# Output as JSON
+python -m tools.migrate flow.flowfile --format json
+
+# Dry run
+python -m tools.migrate ./flows/ --dry-run
+
+# Verbose (show tracebacks)
+python -m tools.migrate flow.flowfile -v
+```
+
+## Output Structure
+
+```yaml
+_version: '2.0'
+_migrated_from: pickle
+flow_id: 1
+flow_name: my_analysis
+flow_settings:
+  name: my_analysis
+  description: null
+  execution_mode: Development
+nodes:
+  - id: 1
+    type: read
+    position: {x: 100, y: 200}
+    settings:
+      received_file:
+        path: data/input.csv
+        file_type: csv
+connections:
+  - [1, 2]
+node_starts:
+  - 1
+```

tools/migrate/__init__.py

@@ -0,0 +1,12 @@
+"""
+Flowfile Migration Tool
+
+Converts old pickle-based .flowfile format to new YAML format.
+
+Usage:
+    python -m tools.migrate <path>
+    python -m tools.migrate old_flow.flowfile
+    python -m tools.migrate ./flows/  # migrate entire directory
+"""
+
+__version__ = "1.0.0"

tools/migrate/__main__.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+"""
+Flowfile Migration Tool - CLI Entry Point
+
+Converts old pickle-based .flowfile format to new YAML format.
+
+Usage:
+    python -m tools.migrate <path> [options]
+
+Examples:
+    # Migrate a single file
+    python -m tools.migrate my_flow.flowfile
+
+    # Migrate to specific output
+    python -m tools.migrate my_flow.flowfile -o my_flow.yaml
+
+    # Migrate entire directory
+    python -m tools.migrate ./flows/
+
+    # Migrate to JSON instead of YAML
+    python -m tools.migrate my_flow.flowfile --format json
+
+    # Migrate directory to different output location
+    python -m tools.migrate ./old_flows/ -o ./new_flows/
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+from tools.migrate.migrate import migrate_directory, migrate_flowfile
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="flowfile-migrate",
+        description="Migrate old .flowfile pickles to YAML format",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s my_flow.flowfile              Migrate single file to YAML
+  %(prog)s ./flows/                      Migrate all files in directory
+  %(prog)s flow.flowfile -o flow.yaml    Specify output path
+  %(prog)s ./flows/ --format json        Output as JSON instead of YAML
+        """,
+    )
+
+    parser.add_argument("path", type=Path, help="Path to .flowfile or directory containing .flowfile files")
+
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        default=None,
+        help="Output path (file or directory). Default: same location with new extension",
+    )
+
+    parser.add_argument(
+        "-f", "--format", choices=["yaml", "json"], default="yaml", help="Output format (default: yaml)"
+    )
+
+    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
+
+    parser.add_argument("--dry-run", action="store_true", help="Show what would be migrated without actually migrating")
+
+    args = parser.parse_args()
+
+    # Validate input path
+    if not args.path.exists():
+        print(f"Error: Path not found: {args.path}", file=sys.stderr)
+        sys.exit(1)
+
+    # Dry run mode
+    if args.dry_run:
+        if args.path.is_file():
+            print(f"Would migrate: {args.path}")
+            suffix = ".yaml" if args.format == "yaml" else ".json"
+            output = args.output or args.path.with_suffix(suffix)
+            print(f"  → {output}")
+        else:
+            flowfiles = list(args.path.glob("**/*.flowfile"))
+            print(f"Would migrate {len(flowfiles)} file(s):")
+            for f in flowfiles:
+                print(f"  - {f}")
+        sys.exit(0)
+
+    # Check for yaml dependency
+    if args.format == "yaml":
+        try:
+            import yaml
+        except ImportError:
+            print("Error: PyYAML is required for YAML output.", file=sys.stderr)
+            print("Install with: pip install pyyaml", file=sys.stderr)
+            sys.exit(1)
+
+    # Run migration
+    try:
+        if args.path.is_file():
+            migrate_flowfile(args.path, args.output, args.format)
+        elif args.path.is_dir():
+            migrate_directory(args.path, args.output, args.format)
+        else:
+            print(f"Error: {args.path} is neither a file nor a directory", file=sys.stderr)
+            sys.exit(1)
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        if args.verbose:
+            import traceback
+
+            traceback.print_exc()
+        sys.exit(1)
+
+    print("\nMigration complete!")
+
+
+if __name__ == "__main__":
+    main()