Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_worker/spawner.py

@@ -1,15 +1,14 @@
-from flowfile_worker import status_dict
-from time import sleep
 import gc
-from typing import List, Tuple
 from multiprocessing import Process, Queue
+from time import sleep
+
+from flowfile_worker import funcs, models, mp_context, status_dict, status_dict_lock
 from flowfile_worker.process_manager import ProcessManager
-from flowfile_worker import models, mp_context, funcs, status_dict_lock
 
 # Initialize ProcessManager
 process_manager = ProcessManager()
 
-flowfile_node_id_type = int|str
+flowfile_node_id_type = int | str
 
 
 def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_message: mp_context.Array, q: Queue):
@@ -48,7 +47,7 @@ def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_mess
                 with status_dict_lock:
                     status_dict[task_id].status = "Error"
                     with error_message.get_lock():
-                        status_dict[task_id].error_message = error_message.value.decode().rstrip('\x00')
+                        status_dict[task_id].error_message = error_message.value.decode().rstrip("\x00")
                 break
 
         p.join()
@@ -72,11 +71,15 @@ def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_mess
         gc.collect()
 
 
-def start_process(polars_serializable_object: bytes, task_id: str,
-                  operation: models.OperationType,
-                  file_ref: str, flowfile_flow_id: int,
-                  flowfile_node_id: flowfile_node_id_type,
-                  kwargs: dict = None) -> None:
+def start_process(
+    polars_serializable_object: bytes,
+    task_id: str,
+    operation: models.OperationType,
+    file_ref: str,
+    flowfile_flow_id: int,
+    flowfile_node_id: flowfile_node_id_type,
+    kwargs: dict = None,
+) -> None:
     """
     Starts a new process for handling Polars dataframe operations.
 
@@ -97,24 +100,31 @@ def start_process(polars_serializable_object: bytes, task_id: str,
     if kwargs is None:
         kwargs = {}
     process_task = getattr(funcs, operation)
-    kwargs['polars_serializable_object'] = polars_serializable_object
-    kwargs['progress'] = mp_context.Value('i', 0)
-    kwargs['error_message'] = mp_context.Array('c', 1024)
-    kwargs['queue'] = Queue(maxsize=1)
-    kwargs['file_path'] = file_ref
-    kwargs['flowfile_flow_id'] = flowfile_flow_id
-    kwargs['flowfile_node_id'] = flowfile_node_id
+    kwargs["polars_serializable_object"] = polars_serializable_object
+    kwargs["progress"] = mp_context.Value("i", 0)
+    kwargs["error_message"] = mp_context.Array("c", 1024)
+    kwargs["queue"] = Queue(maxsize=1)
+    kwargs["file_path"] = file_ref
+    kwargs["flowfile_flow_id"] = flowfile_flow_id
+    kwargs["flowfile_node_id"] = flowfile_node_id
 
     p: Process = mp_context.Process(target=process_task, kwargs=kwargs)
     p.start()
 
     process_manager.add_process(task_id, p)
-    handle_task(task_id=task_id, p=p, progress=kwargs['progress'], error_message=kwargs['error_message'], q=kwargs['queue'])
-
-
-def start_generic_process(func_ref: callable, task_id: str,
-                          file_ref: str, flowfile_flow_id: int,
-                          flowfile_node_id: flowfile_node_id_type, kwargs: dict = None) -> None:
+    handle_task(
+        task_id=task_id, p=p, progress=kwargs["progress"], error_message=kwargs["error_message"], q=kwargs["queue"]
+    )
+
+
+def start_generic_process(
+    func_ref: callable,
+    task_id: str,
+    file_ref: str,
+    flowfile_flow_id: int,
+    flowfile_node_id: flowfile_node_id_type,
+    kwargs: dict = None,
+) -> None:
     """
     Starts a new process for handling generic function execution.
 
@@ -132,30 +142,33 @@ def start_generic_process(func_ref: callable, task_id: str,
         - Delegates to handle_task for process monitoring
     """
     kwargs = {} if kwargs is None else kwargs
-    kwargs['func'] = func_ref
-    kwargs['progress'] = mp_context.Value('i', 0)
-    kwargs['error_message'] = mp_context.Array('c', 1024)
-    kwargs['queue'] = Queue(maxsize=1)
-    kwargs['file_path'] = file_ref
-    kwargs['flowfile_flow_id'] = flowfile_flow_id
-    kwargs['flowfile_node_id'] = flowfile_node_id
-
-    process_task = getattr(funcs, 'generic_task')
+    kwargs["func"] = func_ref
+    kwargs["progress"] = mp_context.Value("i", 0)
+    kwargs["error_message"] = mp_context.Array("c", 1024)
+    kwargs["queue"] = Queue(maxsize=1)
+    kwargs["file_path"] = file_ref
+    kwargs["flowfile_flow_id"] = flowfile_flow_id
+    kwargs["flowfile_node_id"] = flowfile_node_id
+
+    process_task = funcs.generic_task
     p: Process = mp_context.Process(target=process_task, kwargs=kwargs)
     p.start()
 
     process_manager.add_process(task_id, p)  # Add process to process manager
-    handle_task(task_id=task_id, p=p, progress=kwargs['progress'],
-                error_message=kwargs['error_message'], q=kwargs['queue'])
-
-
-def start_fuzzy_process(left_serializable_object: bytes,
-                        right_serializable_object: bytes,
-                        file_ref: str,
-                        fuzzy_maps: List[models.FuzzyMapping],
-                        task_id: str,
-                        flowfile_flow_id: int,
-                        flowfile_node_id: flowfile_node_id_type) -> None:
+    handle_task(
+        task_id=task_id, p=p, progress=kwargs["progress"], error_message=kwargs["error_message"], q=kwargs["queue"]
+    )
+
+
+def start_fuzzy_process(
+    left_serializable_object: bytes,
+    right_serializable_object: bytes,
+    file_ref: str,
+    fuzzy_maps: list[models.FuzzyMapping],
+    task_id: str,
+    flowfile_flow_id: int,
+    flowfile_node_id: flowfile_node_id_type,
+) -> None:
     """
     Starts a new process for performing fuzzy joining operations on two datasets.
 
@@ -172,13 +185,31 @@ def start_fuzzy_process(left_serializable_object: bytes,
         - Initializes and starts a new process for fuzzy joining operation
         - Delegates to handle_task for process monitoring
     """
-    progress = mp_context.Value('i', 0)
-    error_message = mp_context.Array('c', 1024)
+    progress = mp_context.Value("i", 0)
+    error_message = mp_context.Array("c", 1024)
     q = Queue(maxsize=1)
 
-    args: Tuple[bytes, bytes, List[models.FuzzyMapping], mp_context.Array, str, mp_context.Value, Queue, int, flowfile_node_id_type] = \
-        (left_serializable_object, right_serializable_object, fuzzy_maps, error_message, file_ref, progress, q,
-         flowfile_flow_id, flowfile_node_id)
+    args: tuple[
+        bytes,
+        bytes,
+        list[models.FuzzyMapping],
+        mp_context.Array,
+        str,
+        mp_context.Value,
+        Queue,
+        int,
+        flowfile_node_id_type,
+    ] = (
+        left_serializable_object,
+        right_serializable_object,
+        fuzzy_maps,
+        error_message,
+        file_ref,
+        progress,
+        q,
+        flowfile_flow_id,
+        flowfile_node_id,
+    )
 
     p: Process = mp_context.Process(target=funcs.fuzzy_join_task, args=args)
     p.start()

flowfile_worker/utils.py

@@ -1,6 +1,7 @@
+from dataclasses import dataclass
+
 import polars as pl
 from polars.exceptions import PanicException
-from dataclasses import dataclass
 
 
 def collect_lazy_frame(lf: pl.LazyFrame) -> pl.DataFrame:
@@ -12,7 +13,7 @@ def collect_lazy_frame(lf: pl.LazyFrame) -> pl.DataFrame:
 
 @dataclass
 class CollectStreamingInfo:
-    __slots__ = 'df', 'streaming_collect_available'
+    __slots__ = "df", "streaming_collect_available"
     df: pl.DataFrame
     streaming_collect_available: bool
 

shared/__init__.py

@@ -5,11 +5,6 @@ flowfile_core, flowfile_worker, and other components without creating
 circular dependencies.
 """
 
-from .storage_config import storage, get_cache_directory, get_temp_directory, get_flows_directory
+from .storage_config import get_cache_directory, get_flows_directory, get_temp_directory, storage
 
-__all__ = [
-    'storage',
-    'get_cache_directory', 
-    'get_temp_directory',
-    'get_flows_directory'
-]
+__all__ = ["storage", "get_cache_directory", "get_temp_directory", "get_flows_directory"]

shared/storage_config.py

@@ -3,28 +3,40 @@
 Centralized storage configuration for Flowfile.
 This module can be imported by both core and worker without creating dependencies.
 """
+
 import os
 from pathlib import Path
-from typing import Optional, Literal
+from typing import Literal
+
+DirectoryOptions = Literal[
+    "temp_directory",
+    "logs_directory",
+    "system_logs_directory",
+    "database_directory",
+    "cache_directory",
+    "flows_directory",
+    "user_defined_nodes_directory",
+]
+
 
-DirectoryOptions = Literal["temp_directory", "logs_directory",
-                            "system_logs_directory", "database_directory",
-                            "cache_directory", "flows_directory", "user_defined_nodes_directory"]
+def _is_docker_mode() -> bool:
+    """Check if running in Docker mode based on FLOWFILE_MODE."""
+    return os.environ.get("FLOWFILE_MODE") == "docker"
 
 
 class FlowfileStorage:
     """Centralized storage manager for Flowfile applications."""
 
     def __init__(self):
-        self._base_dir: Optional[Path] = None
-        self._user_data_dir: Optional[Path] = None
+        self._base_dir: Path | None = None
+        self._user_data_dir: Path | None = None
         self._ensure_directories()
 
     @property
     def base_directory(self) -> Path:
         """Get the base Flowfile storage directory (for internal container communication)."""
         if self._base_dir is None:
-            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            if _is_docker_mode():
                 # In Docker, internal storage stays inside /app
                 base_path = os.environ.get("FLOWFILE_STORAGE_DIR", "/app/internal_storage")
             else:
@@ -41,7 +53,7 @@ class FlowfileStorage:
     def user_data_directory(self) -> Path:
         """Get the user data directory (completely separate from application code)."""
         if self._user_data_dir is None:
-            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            if _is_docker_mode():
                 # In Docker, user data is at /data/user (completely outside /app)
                 user_data_path = os.environ.get("FLOWFILE_USER_DATA_DIR", "/data/user")
             else:
@@ -70,7 +82,7 @@ class FlowfileStorage:
     @property
     def flows_directory(self) -> Path:
         """Directory for flow storage (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             # In Docker, flows are in separate user data area
             return self.user_data_directory / "flows"
         else:
@@ -80,7 +92,7 @@ class FlowfileStorage:
     @property
     def uploads_directory(self) -> Path:
         """Directory for user uploads (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             # In Docker, uploads are in separate user data area
             return self.user_data_directory / "uploads"
         else:
@@ -90,7 +102,7 @@ class FlowfileStorage:
     @property
     def user_defined_nodes_directory(self) -> Path:
         """Directory for user-defined custom nodes (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             return self.user_data_directory / "user_defined_nodes"
         else:
             return self.base_directory / "user_defined_nodes"
@@ -103,7 +115,7 @@ class FlowfileStorage:
     @property
     def outputs_directory(self) -> Path:
         """Directory for user outputs (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             # In Docker, outputs are in separate user data area
             return self.user_data_directory / "outputs"
         else:
@@ -184,8 +196,8 @@ class FlowfileStorage:
 
     def cleanup_directory(self, directory_option: DirectoryOptions, storage_duration_hours: int = 24) -> None:
         """Clean up any directory of the folder"""
-        import time
         import shutil
+        import time
 
         if not hasattr(self, directory_option):
             raise Exception(f"Directory does not exist in {self.base_directory}")

test_utils/postgres/commands.py

@@ -6,8 +6,9 @@ to start and stop PostgreSQL containers with sample data.
 """
 
 import argparse
-import sys
 import logging
+import sys
+
 from . import fixtures
 
 # Set up logging
@@ -106,4 +107,4 @@ if __name__ == "__main__":
         sys.exit(stop_postgres())
     else:
         print("Usage: python -m test_utils.postgres.commands [start|stop] [options]")
-        sys.exit(1)
+        sys.exit(1)

test_utils/postgres/fixtures.py

@@ -5,13 +5,13 @@ This module provides utilities to set up, manage, and tear down PostgreSQL
 containers with sample data for testing.
 """
 
-import os
-import time
 import logging
-import subprocess
+import os
 import shutil
+import subprocess
+import time
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Dict, Generator, Optional, Tuple
 
 # Configure logging
 logging.basicConfig(
@@ -208,7 +208,7 @@ def start_postgres_container(
         container_name: str = POSTGRES_CONTAINER_NAME,
         port: int = POSTGRES_PORT,
         image_tag: str = POSTGRES_IMAGE_TAG
-) -> Tuple[Optional[subprocess.Popen], bool]:
+) -> tuple[subprocess.Popen | None, bool]:
     """
     Start the PostgreSQL container with sample data.
 
@@ -225,7 +225,7 @@ def start_postgres_container(
         logger.warning("Docker not available, skipping PostgreSQL container start")
         return None, False
 
-    logger.info(f"Starting PostgreSQL container with sample data...")
+    logger.info("Starting PostgreSQL container with sample data...")
 
     # Check if container is already running
     if is_container_running(container_name):
@@ -345,12 +345,12 @@ def print_connection_info(
     print(f"Connection string: postgresql://{user}:{password}@{host}:{port}/{db}")
     print("=" * 50)
     print("\nTo stop the container, run:")
-    print(f"poetry run stop_postgres")
+    print("poetry run stop_postgres")
     print("=" * 50 + "\n")
 
 
 @contextmanager
-def managed_postgres() -> Generator[Dict[str, any], None, None]:
+def managed_postgres() -> Generator[dict[str, any], None, None]:
     """
     Context manager for PostgreSQL container management.
     Ensures proper cleanup even when tests fail.
@@ -414,4 +414,4 @@ def get_db_engine():
         return engine
     except ImportError:
         logger.error("SQLAlchemy not installed. Run: pip install sqlalchemy")
-        raise
+        raise

test_utils/s3/commands.py

@@ -43,4 +43,4 @@ def stop_minio():
     if fixtures.stop_minio_container():
         print("MinIO stopped successfully")
         return 0
-    return 1
+    return 1

test_utils/s3/data_generator.py

@@ -1,17 +1,16 @@
 
-import logging
 import io
+import logging
 import os
 
 # Third-party libraries
 import boto3
-from botocore.client import Config
 import polars as pl
 import pyarrow as pa
+from botocore.client import Config
 from deltalake import write_deltalake
 from pyiceberg.catalog import load_catalog
 
-
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -289,4 +288,4 @@ if __name__ == '__main__':
     populate_test_data(endpoint_url=MINIO_ENDPOINT_URL,
                        access_key=MINIO_ACCESS_KEY,
                        secret_key=MINIO_SECRET_KEY,
-                       bucket_name="test-bucket")
+                       bucket_name="test-bucket")

test_utils/s3/demo_data_generator.py

@@ -1,17 +1,14 @@
-import logging
 import io
+import logging
 import os
-import tempfile
-import shutil
 import random
+import tempfile
 from datetime import datetime, timedelta
 
 # Third-party libraries
 import boto3
-from botocore.client import Config
 import polars as pl
-import pyarrow as pa
-from pyarrow import parquet as pq
+from botocore.client import Config
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -152,7 +149,7 @@ def create_demo_data(endpoint_url: str, access_key: str, secret_key: str, bucket
     unique_product_ids = sales_df["product_id"].unique().to_list()
     # Create a map of product_id to unit_price from the first occurrence in sales_df
     product_price_map = sales_df.group_by("product_id").agg(pl.first("unit_price")).to_dict(as_series=False)
-    price_dict = dict(zip(product_price_map['product_id'], product_price_map['unit_price']))
+    price_dict = dict(zip(product_price_map['product_id'], product_price_map['unit_price'], strict=False))
 
     products_df = pl.DataFrame({
         "product_id": unique_product_ids,

test_utils/s3/fixtures.py

@@ -1,12 +1,14 @@
+import logging
 import os
-import time
+import shutil
 import subprocess
-import logging
+import time
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Dict, Generator
-import shutil
+
 import boto3
 from botocore.client import Config
+
 from test_utils.s3.data_generator import populate_test_data
 from test_utils.s3.demo_data_generator import create_demo_data
 
@@ -191,7 +193,7 @@ def start_minio_container() -> bool:
 
 
 @contextmanager
-def managed_minio() -> Generator[Dict[str, any], None, None]:
+def managed_minio() -> Generator[dict[str, any], None, None]:
     """Context manager for MinIO container with full connection info"""
     if not start_minio_container():
         yield {}

tools/migrate/__init__.py

@@ -9,4 +9,4 @@ Usage:
     python -m tools.migrate ./flows/  # migrate entire directory
 """
 
-__version__ = "1.0.0"
+__version__ = "1.0.0"

tools/migrate/__main__.py

@@ -28,13 +28,13 @@ import argparse
 import sys
 from pathlib import Path
 
-from tools.migrate.migrate import migrate_flowfile, migrate_directory
+from tools.migrate.migrate import migrate_directory, migrate_flowfile
 
 
 def main():
     parser = argparse.ArgumentParser(
-        prog='flowfile-migrate',
-        description='Migrate old .flowfile pickles to YAML format',
+        prog="flowfile-migrate",
+        description="Migrate old .flowfile pickles to YAML format",
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
 Examples:
@@ -42,40 +42,26 @@ Examples:
   %(prog)s ./flows/                      Migrate all files in directory
   %(prog)s flow.flowfile -o flow.yaml    Specify output path
   %(prog)s ./flows/ --format json        Output as JSON instead of YAML
-        """
+        """,
     )
 
-    parser.add_argument(
-        'path',
-        type=Path,
-        help='Path to .flowfile or directory containing .flowfile files'
-    )
+    parser.add_argument("path", type=Path, help="Path to .flowfile or directory containing .flowfile files")
 
     parser.add_argument(
-        '-o', '--output',
+        "-o",
+        "--output",
         type=Path,
         default=None,
-        help='Output path (file or directory). Default: same location with new extension'
+        help="Output path (file or directory). Default: same location with new extension",
     )
 
     parser.add_argument(
-        '-f', '--format',
-        choices=['yaml', 'json'],
-        default='yaml',
-        help='Output format (default: yaml)'
+        "-f", "--format", choices=["yaml", "json"], default="yaml", help="Output format (default: yaml)"
     )
 
-    parser.add_argument(
-        '-v', '--verbose',
-        action='store_true',
-        help='Verbose output'
-    )
+    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
 
-    parser.add_argument(
-        '--dry-run',
-        action='store_true',
-        help='Show what would be migrated without actually migrating'
-    )
+    parser.add_argument("--dry-run", action="store_true", help="Show what would be migrated without actually migrating")
 
     args = parser.parse_args()
 
@@ -88,18 +74,18 @@ Examples:
     if args.dry_run:
         if args.path.is_file():
             print(f"Would migrate: {args.path}")
-            suffix = '.yaml' if args.format == 'yaml' else '.json'
+            suffix = ".yaml" if args.format == "yaml" else ".json"
             output = args.output or args.path.with_suffix(suffix)
             print(f"  → {output}")
         else:
-            flowfiles = list(args.path.glob('**/*.flowfile'))
+            flowfiles = list(args.path.glob("**/*.flowfile"))
             print(f"Would migrate {len(flowfiles)} file(s):")
             for f in flowfiles:
                 print(f"  - {f}")
         sys.exit(0)
 
     # Check for yaml dependency
-    if args.format == 'yaml':
+    if args.format == "yaml":
         try:
             import yaml
         except ImportError:
@@ -121,11 +107,12 @@ Examples:
         print(f"Error: {e}", file=sys.stderr)
         if args.verbose:
             import traceback
+
             traceback.print_exc()
         sys.exit(1)
 
     print("\nMigration complete!")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()