Flowfile 0.5.1__tar.gz → 0.5.6__tar.gz

{flowfile-0.5.1 → flowfile-0.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Flowfile
-Version: 0.5.1
+Version: 0.5.6
 Summary: Project combining flowfile core (backend) and flowfile_worker (compute offloader) and flowfile_frame (api)
 License-File: LICENSE
 Author: Edward van Eechoud
@@ -33,16 +33,15 @@ Requires-Dist: polars (>=1.8.2,<1.32) ; sys_platform != "win32"
 Requires-Dist: polars (>=1.8.2,<=1.25.2) ; sys_platform == "win32"
 Requires-Dist: polars-distance (>=0.4.3,<0.5.0)
 Requires-Dist: polars-ds (>=0.6.0)
-Requires-Dist: polars-expr-transformer (>=0.4.9.0)
+Requires-Dist: polars-expr-transformer (>=0.5.0)
 Requires-Dist: polars-grouper (>=0.3.0,<0.4.0)
 Requires-Dist: polars_simed (>=0.3.4,<0.4.0)
 Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
 Requires-Dist: pydantic (>=2.9.2,<2.10.0)
 Requires-Dist: pyiceberg[hadoop] (>=0.9.1,<0.10.0)
-Requires-Dist: pyinstaller (>=6.11.0,<7.0.0)
-Requires-Dist: pytest (>=8.3.4,<9.0.0)
 Requires-Dist: python-jose (>=3.4.0,<4.0.0)
 Requires-Dist: python-multipart (>=0.0.12,<0.1.0)
+Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
 Requires-Dist: s3fs (>=2025.7.0,<2026.0.0)
 Requires-Dist: tqdm (>=4.67.1,<5.0.0)
 Requires-Dist: uvicorn (>=0.32.0,<0.33.0)

{flowfile-0.5.1 → flowfile-0.5.6}/build_backends/build_backends/main.py

@@ -1,12 +1,11 @@
 import os
-import subprocess
 import platform
 import shutil
+import subprocess
 from pathlib import Path
-from typing import List
 
 
-def merge_directories(directories: List[str], target_dir: str, cleanup_after_merge: bool = True):
+def merge_directories(directories: list[str], target_dir: str, cleanup_after_merge: bool = True):
     """
     Merge all files from two folders into a new target directory.
     After successful merge, removes the original folders.
@@ -16,9 +15,9 @@ def merge_directories(directories: List[str], target_dir: str, cleanup_after_mer
     for directory in directories:
         if os.path.exists(directory):
             shutil.copytree(directory, target_dir, dirs_exist_ok=True)
-    print('Merged directories:', directories, 'into', target_dir)
+    print("Merged directories:", directories, "into", target_dir)
     if cleanup_after_merge:
-        print('Cleaning up directories:', directories)
+        print("Cleaning up directories:", directories)
         for directory in directories:
             if os.path.exists(directory):
                 shutil.rmtree(directory)
@@ -197,25 +196,28 @@ coll = COLLECT(
     name='{output_name}'
 )
 '''
-    spec_path = f'{output_name}.spec'
-    with open(spec_path, 'w') as f:
+    spec_path = f"{output_name}.spec"
+    with open(spec_path, "w") as f:
         f.write(spec_content)
     return spec_path
 
+
 def build_backend(directory, script_name, output_name, hidden_imports=None):
     try:
         spec_path = create_spec_file(directory, script_name, output_name, hidden_imports)
 
         env = os.environ.copy()
-        env['PYTHONOPTIMIZE'] = "1"
+        env["PYTHONOPTIMIZE"] = "1"
 
         command = [
             "pyinstaller",
             "--clean",
             "-y",
-            "--dist", "./services_dist",
-            "--workpath", "/tmp" if platform.system() != "Windows" else os.path.join(os.getenv('TEMP'), 'pyinstaller'),
-            spec_path
+            "--dist",
+            "./services_dist",
+            "--workpath",
+            "/tmp" if platform.system() != "Windows" else os.path.join(os.getenv("TEMP"), "pyinstaller"),
+            spec_path,
         ]
 
         print(f"Building {output_name}...")
@@ -247,7 +249,7 @@ def combine_packages():
             exe_name = project + ".exe" if platform.system() == "Windows" else project
             src_exe = os.path.join(src_dir, exe_name)
             temp_target_exe = os.path.join(dist_dir, "_" + exe_name)
-            target_exe = os.path.join(dist_dir,  exe_name)
+            target_exe = os.path.join(dist_dir, exe_name)
             if os.path.exists(src_exe) and os.path.isfile(src_exe):
                 # Instead of removing, overwrite the target
                 shutil.move(src_exe, temp_target_exe)
@@ -257,9 +259,10 @@ def combine_packages():
             if platform.system() == "Windows" and os.path.exists(os.path.join(dist_dir, project)):
                 shutil.rmtree(os.path.join(dist_dir, project))
 
+
 def main():
     # Clean previous builds
-    for dir_name in ['services_dist']:
+    for dir_name in ["services_dist"]:
         if os.path.exists(dir_name):
             shutil.rmtree(dir_name)
 
@@ -286,20 +289,20 @@ def main():
     # Build flowfile_worker
 
     if not build_backend(
-            directory=os.path.join("flowfile_worker", "flowfile_worker"),
-            script_name="main.py",
-            output_name="flowfile_worker",
-            hidden_imports=common_imports
+        directory=os.path.join("flowfile_worker", "flowfile_worker"),
+        script_name="main.py",
+        output_name="flowfile_worker",
+        hidden_imports=common_imports,
     ):
         builds_successful = False
 
     # Build flowfile_core
 
     if not build_backend(
-            directory=os.path.join("flowfile_core", "flowfile_core"),
-            script_name="main.py",
-            output_name="flowfile_core",
-            hidden_imports=common_imports
+        directory=os.path.join("flowfile_core", "flowfile_core"),
+        script_name="main.py",
+        output_name="flowfile_core",
+        hidden_imports=common_imports,
     ):
         builds_successful = False
 
@@ -310,4 +313,4 @@ def main():
 
 
 # if __name__ == "__main__":
-#     main()
+#     main()

{flowfile-0.5.1 → flowfile-0.5.6}/build_backends/build_backends/main_prd.py

@@ -1,14 +1,9 @@
-import os
-import subprocess
-import platform
-from concurrent.futures import ProcessPoolExecutor, wait
-
 import subprocess
 import time
-import requests
-from statistics import mean, stdev
-import sys
 from datetime import datetime
+from statistics import mean, stdev
+
+import requests
 
 
 def wait_for_endpoint(url, timeout=60):
@@ -27,11 +22,7 @@ def wait_for_endpoint(url, timeout=60):
 def shutdown_service():
     """Shutdown the service gracefully using the shutdown endpoint."""
     try:
-        response = requests.post(
-            'http://0.0.0.0:63578/shutdown',
-            headers={'accept': 'application/json'},
-            data=''
-        )
+        response = requests.post("http://0.0.0.0:63578/shutdown", headers={"accept": "application/json"}, data="")
         print("Shutdown request sent, waiting for service to stop...")
         time.sleep(1)  # Wait 10 seconds to ensure the service is fully stopped
         return True
@@ -97,14 +88,14 @@ def run_comparison_test(old_exe, new_exe, num_runs=3):
     print("\nResults:")
     print("-" * 50)
     if old_times:
-        print(f"Old executable:")
+        print("Old executable:")
         print(f"  Average: {mean(old_times):.3f} seconds")
         print(f"  Std Dev: {stdev(old_times):.3f} seconds" if len(old_times) > 1 else "  Std Dev: N/A")
         print(f"  Min: {min(old_times):.3f} seconds")
         print(f"  Max: {max(old_times):.3f} seconds")
 
     if new_times:
-        print(f"\nNew executable:")
+        print("\nNew executable:")
         print(f"  Average: {mean(new_times):.3f} seconds")
         print(f"  Std Dev: {stdev(new_times):.3f} seconds" if len(new_times) > 1 else "  Std Dev: N/A")
         print(f"  Min: {min(new_times):.3f} seconds")
@@ -112,13 +103,13 @@ def run_comparison_test(old_exe, new_exe, num_runs=3):
 
     if old_times and new_times:
         improvement = (mean(old_times) - mean(new_times)) / mean(old_times) * 100
-        print(f"\nPerformance difference:")
+        print("\nPerformance difference:")
         print(f"  {improvement:.1f}% {'faster' if improvement > 0 else 'slower'} than old version")
 
 
 if __name__ == "__main__":
-    old_exe = '/Users/edwardvanechoud/personal_dev/Flowfile/dist/flowfile_core/flowfile_core'
-    new_exe = '/Users/edwardvanechoud/personal_dev/Flowfile/dist_flowfile_core/flowfile_core'
+    old_exe = "/Users/edwardvanechoud/personal_dev/Flowfile/dist/flowfile_core/flowfile_core"
+    new_exe = "/Users/edwardvanechoud/personal_dev/Flowfile/dist_flowfile_core/flowfile_core"
 
     run_comparison_test(old_exe, old_exe)
 
@@ -199,4 +190,4 @@ if __name__ == "__main__":
 #
 #
 # if __name__ == "__main__":
-#     main()
+#     main()

flowfile-0.5.6/flowfile/flowfile/__init__.py

@@ -0,0 +1,228 @@
+"""
+FlowFile: A framework combining visual ETL with a Polars-like API.
+
+This package ties together the FlowFile ecosystem components:
+- flowfile_core: Core ETL functionality
+- flowfile_frame: Polars-like DataFrame API
+- flowfile_worker: Computation engine
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("Flowfile")
+except PackageNotFoundError:
+    __version__ = "0.5.0"
+
+import logging
+import os
+
+os.environ["FLOWFILE_WORKER_PORT"] = "63578"
+os.environ["FLOWFILE_SINGLE_FILE_MODE"] = "1"
+
+from polars.datatypes import (
+    Array,
+    Binary,
+    Boolean,
+    Categorical,
+    DataType,
+    DataTypeClass,
+    Date,
+    Datetime,
+    Decimal,
+    Duration,
+    Enum,
+    Field,
+    Float32,
+    Float64,
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    Int128,
+    List,
+    Null,
+    Object,
+    String,
+    Struct,
+    Time,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Unknown,
+    Utf8,
+)
+
+from flowfile.api import open_graph_in_editor
+from flowfile.web import start_server as start_web_ui
+from flowfile_core.flowfile import node_designer
+from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.flowfile.flow_node.flow_node import FlowNode
+from flowfile_core.schemas import input_schema as node_interface
+from flowfile_core.schemas import transform_schema
+from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection
+from flowfile_core.schemas.schemas import FlowInformation, FlowSettings
+from flowfile_frame import (
+    FuzzyMapping,
+    concat,
+    create_cloud_storage_connection,
+    create_cloud_storage_connection_if_not_exists,
+    create_database_connection,
+    create_database_connection_if_not_exists,
+    del_cloud_storage_connection,
+    del_database_connection,
+    from_dict,
+    get_all_available_cloud_storage_connections,
+    get_all_available_database_connections,
+    get_database_connection_by_name,
+    read_csv,
+    read_database,
+    read_parquet,
+    scan_csv,
+    scan_csv_from_cloud_storage,
+    scan_delta,
+    scan_json_from_cloud_storage,
+    scan_parquet,
+    scan_parquet_from_cloud_storage,
+    write_database,
+)
+from flowfile_frame.expr import col, column, count, cum_count, len, lit, max, mean, min, sum, when
+from flowfile_frame.flow_frame import FlowFrame
+from flowfile_frame.group_frame import GroupByFrame
+from flowfile_frame.selectors import (
+    all_,
+    boolean,
+    by_dtype,
+    categorical,
+    contains,
+    date,
+    datetime,
+    duration,
+    ends_with,
+    float_,
+    integer,
+    list_,
+    matches,
+    numeric,
+    object_,
+    starts_with,
+    string,
+    struct,
+    temporal,
+    time,
+)
+from flowfile_frame.utils import create_flow_graph
+
+__all__ = [
+    # Core FlowFrame classes
+    "FlowFrame",
+    "GroupByFrame",
+    "FullCloudStorageConnection",
+    # Main creation functions
+    "read_csv",
+    "read_parquet",
+    "from_dict",
+    "concat",
+    "scan_csv",
+    "scan_parquet",
+    "scan_delta",
+    "scan_parquet_from_cloud_storage",
+    "scan_json_from_cloud_storage",
+    "scan_csv_from_cloud_storage",
+    # Cloud storage connection management
+    "get_all_available_cloud_storage_connections",
+    "create_cloud_storage_connection",
+    "del_cloud_storage_connection",
+    "create_cloud_storage_connection_if_not_exists",
+    # Database functions
+    "read_database",
+    "write_database",
+    "create_database_connection",
+    "create_database_connection_if_not_exists",
+    "del_database_connection",
+    "get_all_available_database_connections",
+    "get_database_connection_by_name",
+    "FlowGraph",
+    "FlowDataEngine",
+    "node_interface",
+    "FlowSettings",
+    "transform_schema",
+    "FlowNode",
+    "FlowfileColumn",
+    "FlowInformation",
+    "FuzzyMapping",
+    # Expression API
+    "col",
+    "lit",
+    "column",
+    "cum_count",
+    "len",
+    "sum",
+    "min",
+    "max",
+    "mean",
+    "count",
+    "when",
+    # Selector utilities
+    "numeric",
+    "float_",
+    "integer",
+    "string",
+    "temporal",
+    "datetime",
+    "date",
+    "time",
+    "duration",
+    "boolean",
+    "categorical",
+    "object_",
+    "list_",
+    "struct",
+    "all_",
+    "by_dtype",
+    "contains",
+    "starts_with",
+    "ends_with",
+    "matches",
+    "node_designer",
+    # Utilities
+    "create_flow_graph",
+    "open_graph_in_editor",
+    # Data types from Polars
+    "Int8",
+    "Int16",
+    "Int32",
+    "Int64",
+    "Int128",
+    "UInt8",
+    "UInt16",
+    "UInt32",
+    "UInt64",
+    "Float32",
+    "Float64",
+    "Boolean",
+    "String",
+    "Utf8",
+    "Binary",
+    "Null",
+    "List",
+    "Array",
+    "Struct",
+    "Object",
+    "Date",
+    "Time",
+    "Datetime",
+    "Duration",
+    "Categorical",
+    "Decimal",
+    "Enum",
+    "Unknown",
+    "DataType",
+    "DataTypeClass",
+    "Field",
+    "start_web_ui",
+]
+logging.getLogger("PipelineHandler").setLevel(logging.WARNING)

flowfile-0.5.6/flowfile/flowfile/__main__.py

@@ -0,0 +1,153 @@
+#  flowfile/__main__.py
+
+import sys
+from pathlib import Path
+
+
+def run_flow(flow_path: str) -> int:
+    """
+    Load and execute a flow from a YAML/JSON file.
+
+    Args:
+        flow_path: Path to the flow file (.yaml, .yml, or .json)
+
+    Returns:
+        Exit code (0 for success, 1 for failure)
+    """
+    # Disable worker communication for CLI execution
+    from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
+
+    OFFLOAD_TO_WORKER.set(False)
+
+    from flowfile_core.flowfile.manage.io_flowfile import open_flow
+
+    path = Path(flow_path)
+    if not path.exists():
+        print(f"Error: File not found: {flow_path}", file=sys.stderr)
+        return 1
+
+    if path.suffix.lower() not in (".yaml", ".yml", ".json"):
+        print(f"Error: Unsupported file format: {path.suffix}", file=sys.stderr)
+        print("Supported formats: .yaml, .yml, .json", file=sys.stderr)
+        return 1
+
+    print(f"Loading flow from: {flow_path}")
+
+    try:
+        flow = open_flow(path)
+    except Exception as e:
+        print(f"Error loading flow: {e}", file=sys.stderr)
+        return 1
+
+    # Force local execution for CLI - no worker service needed
+    flow.execution_location = "local"
+
+    # Remove explore_data nodes - they're UI-only and require a worker service
+    explore_data_nodes = [n.node_id for n in flow.nodes if n.node_type == "explore_data"]
+    for node_id in explore_data_nodes:
+        flow.delete_node(node_id)
+    if explore_data_nodes:
+        print(f"Skipping {len(explore_data_nodes)} explore_data node(s) (UI-only)")
+
+    flow_name = flow.flow_settings.name or f"Flow {flow.flow_id}"
+    print(f"Running flow: {flow_name} (id={flow.flow_id})")
+    print(f"Nodes: {len(flow.nodes)}")
+    flow.print_tree()
+    print("-" * 40)
+
+    try:
+        result = flow.run_graph()
+    except Exception as e:
+        print(f"Error running flow: {e}", file=sys.stderr)
+        return 1
+
+    if result is None:
+        print("Error: Flow execution returned no result", file=sys.stderr)
+        return 1
+
+    # Display results
+    print("-" * 40)
+    if result.success:
+        duration = ""
+        if result.start_time and result.end_time:
+            duration = f" in {(result.end_time - result.start_time).total_seconds():.2f}s"
+        print(f"Flow completed successfully{duration}")
+        print(f"Nodes completed: {result.nodes_completed}/{result.number_of_nodes}")
+    else:
+        print("Flow execution failed", file=sys.stderr)
+        for node_result in result.node_step_result:
+            if not node_result.success and node_result.error:
+                node_name = node_result.node_name or f"Node {node_result.node_id}"
+                print(f"  - {node_name}: {node_result.error}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+def main():
+    """
+    Display information about FlowFile when run directly as a module.
+    """
+    import argparse
+
+    import flowfile
+
+    parser = argparse.ArgumentParser(description="FlowFile: A visual ETL tool with a Polars-like API")
+    parser.add_argument("command", nargs="?", choices=["run"], help="Command to execute")
+    parser.add_argument(
+        "component", nargs="?", choices=["ui", "core", "worker", "flow"], help="Component to run"
+    )
+    parser.add_argument("file_path", nargs="?", help="Path to flow file (for 'flow' component)")
+    parser.add_argument("--host", default="127.0.0.1", help="Host to bind the server to")
+    parser.add_argument("--port", type=int, default=63578, help="Port to bind the server to")
+    parser.add_argument("--no-browser", action="store_true", help="Don't open a browser window")
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    if args.command == "run" and args.component:
+        if args.component == "ui":
+            try:
+                flowfile.start_web_ui(open_browser=not args.no_browser)
+            except KeyboardInterrupt:
+                print("\nFlowFile service stopped.")
+        elif args.component == "core":
+            # Only for direct core service usage
+            from flowfile_core.main import run as run_core
+
+            run_core(host=args.host, port=args.port)
+        elif args.component == "worker":
+            # Only for direct worker service usage
+            from flowfile_worker.main import run as run_worker
+
+            run_worker(host=args.host, port=args.port)
+        elif args.component == "flow":
+            if not args.file_path:
+                print("Error: 'flow' component requires a file path", file=sys.stderr)
+                print("Usage: flowfile run flow <path-to-flow-file>", file=sys.stderr)
+                sys.exit(1)
+            sys.exit(run_flow(args.file_path))
+    else:
+        # Default action - show info
+        print(f"FlowFile v{flowfile.__version__}")
+        print("A framework combining visual ETL with a Polars-like API")
+        print("\nUsage:")
+        print("  # Start the FlowFile web UI with integrated services")
+        print("  flowfile run ui")
+        print("")
+        print("  # Run a flow from a file")
+        print("  flowfile run flow my_pipeline.yaml")
+        print("")
+        print("  # Advanced: Run individual components")
+        print("  flowfile run core  # Start only the core service")
+        print("  flowfile run worker  # Start only the worker service")
+        print("")
+        print("  # Options")
+        print("  flowfile run ui --host 0.0.0.0 --port 8080  # Custom host/port")
+        print("  flowfile run ui --no-browser  # Don't open browser")
+        print("")
+        print("  # Python API usage examples")
+        print("  import flowfile as ff")
+        print("  df = ff.read_csv('data.csv')")
+        print("  result = df.filter(ff.col('value') > 10)")
+        print("  ff.open_graph_in_editor(result)")